From 56b9bf40d7ad8dcac6538f115f01e6be2ad51a2d Mon Sep 17 00:00:00 2001 From: donkirkby Date: Fri, 14 Jun 2019 15:48:28 -0700 Subject: [PATCH] Remove run cables and a bunch of old tests, for #752. --- .../{2019_02 => 2019_06}/example_names.csv | 0 kive/archive/management/__init__.py | 0 kive/archive/management/commands/__init__.py | 0 .../commands/check_complete_runs.py | 198 - .../management/commands/checkrcconsistency.py | 39 - .../management/commands/force_reruns.py | 115 - .../management/commands/purge_batch.py | 31 - .../commands/set_old_state_fields.py | 42 - kive/archive/migrations/0108_drop_cables.py | 55 + kive/archive/models.py | 788 +-- kive/archive/tests.py | 4425 --------------- kive/archive/tests_mock.py | 1272 ----- kive/archive/tests_queuedrun.py | 1506 ------ .../commands/consolidate_containers.py | 76 - .../commands/convert_docker_images.py | 115 - .../management/commands/convert_pipelines.py | 435 -- .../management/commands/count_queries.py | 102 + kive/datachecking/tests.py | 156 - kive/fleet/management/__init__.py | 0 kive/fleet/management/commands/__init__.py | 0 .../fleet/management/commands/cable_helper.py | 29 - kive/fleet/management/commands/fleetworker.py | 14 - kive/fleet/management/commands/runfleet.py | 40 - kive/fleet/management/commands/step_helper.py | 50 - kive/fleet/tests.py | 79 - kive/fleet/tests_docker_build.py | 59 - kive/fleet/tests_dockerlib.py | 198 - kive/fleet/tests_idletasks.py | 178 - kive/fleet/tests_slurmlib.py | 700 --- kive/fleet/tests_slurmscheduler.py | 276 - kive/fleet/workers.py | 1206 ----- kive/kive/testing_utils.py | 2332 +------- .../commands/copy_and_check_md5s.py | 207 - .../librarian/migrations/0110_mark_uploads.py | 2 + .../0112_remove_dataset_file_source.py | 19 + kive/librarian/models.py | 16 - kive/librarian/tests.py | 863 +-- kive/librarian/tests_mock.py | 649 +-- kive/librarian/views.py | 5 +- kive/metadata/tests.py | 2814 +--------- kive/method/tests.py | 1768 ------ kive/method/tests_mock.py | 1041 ---- kive/pipeline/tests.py | 3067 ----------- kive/pipeline/tests_mock.py | 834 --- .../archive_no_runs_test_environment.json | 2558 --------- .../fixtures/archive_test_environment.json | 2905 ---------- kive/portal/fixtures/container_run.json | 16 +- kive/portal/fixtures/deep_nested_run.json | 4780 ----------------- kive/portal/fixtures/demo.json | 3471 ------------ .../fixtures/em_sandbox_test_environment.json | 2622 --------- ...ecute_discarded_intermediate_tests_rm.json | 2256 -------- .../fixtures/execute_result_tests_rm.json | 2770 ---------- kive/portal/fixtures/execute_tests.json | 648 --- kive/portal/fixtures/find_datasets.json | 1556 ------ kive/portal/fixtures/removal.json | 1660 ------ .../fixtures/restore_reusable_dataset.json | 1151 ---- kive/portal/fixtures/run_api_tests.json | 1528 ------ .../run_component_too_many_checks.json | 2138 -------- .../run_pipelines_recovering_reused_step.json | 3751 ------------- kive/portal/fixtures/simple_run.json | 3231 ----------- .../commands/update_test_fixtures.py | 1028 ---- kive/sandbox/execute.py | 2631 --------- kive/sandbox/tests.py | 1852 ------- kive/sandbox/tests_rm.py | 613 --- kive/stopwatch/tests.py | 10 +- utils/prepare_initial_pipeline.py | 66 - 66 files changed, 243 insertions(+), 68799 deletions(-) rename kive/FixtureFiles/container_run/Datasets/{2019_02 => 2019_06}/example_names.csv (100%) delete mode 100644 kive/archive/management/__init__.py delete mode 100644 kive/archive/management/commands/__init__.py delete mode 100644 kive/archive/management/commands/check_complete_runs.py delete mode 100644 kive/archive/management/commands/checkrcconsistency.py delete mode 100644 kive/archive/management/commands/force_reruns.py delete mode 100644 kive/archive/management/commands/purge_batch.py delete mode 100644 kive/archive/management/commands/set_old_state_fields.py create mode 100644 kive/archive/migrations/0108_drop_cables.py delete mode 100644 kive/archive/tests.py delete mode 100644 kive/archive/tests_mock.py delete mode 100644 kive/archive/tests_queuedrun.py delete mode 100644 kive/container/management/commands/consolidate_containers.py delete mode 100644 kive/container/management/commands/convert_docker_images.py delete mode 100644 kive/container/management/commands/convert_pipelines.py create mode 100644 kive/container/management/commands/count_queries.py delete mode 100644 kive/datachecking/tests.py delete mode 100644 kive/fleet/management/__init__.py delete mode 100644 kive/fleet/management/commands/__init__.py delete mode 100644 kive/fleet/management/commands/cable_helper.py delete mode 100644 kive/fleet/management/commands/fleetworker.py delete mode 100644 kive/fleet/management/commands/runfleet.py delete mode 100644 kive/fleet/management/commands/step_helper.py delete mode 100644 kive/fleet/tests.py delete mode 100644 kive/fleet/tests_docker_build.py delete mode 100755 kive/fleet/tests_dockerlib.py delete mode 100644 kive/fleet/tests_idletasks.py delete mode 100644 kive/fleet/tests_slurmlib.py delete mode 100644 kive/fleet/tests_slurmscheduler.py delete mode 100644 kive/fleet/workers.py delete mode 100644 kive/librarian/management/commands/copy_and_check_md5s.py create mode 100644 kive/librarian/migrations/0112_remove_dataset_file_source.py delete mode 100644 kive/method/tests.py delete mode 100644 kive/method/tests_mock.py delete mode 100644 kive/pipeline/tests.py delete mode 100644 kive/pipeline/tests_mock.py delete mode 100644 kive/portal/fixtures/archive_no_runs_test_environment.json delete mode 100644 kive/portal/fixtures/archive_test_environment.json delete mode 100644 kive/portal/fixtures/deep_nested_run.json delete mode 100644 kive/portal/fixtures/demo.json delete mode 100644 kive/portal/fixtures/em_sandbox_test_environment.json delete mode 100644 kive/portal/fixtures/execute_discarded_intermediate_tests_rm.json delete mode 100644 kive/portal/fixtures/execute_result_tests_rm.json delete mode 100644 kive/portal/fixtures/execute_tests.json delete mode 100644 kive/portal/fixtures/find_datasets.json delete mode 100644 kive/portal/fixtures/removal.json delete mode 100644 kive/portal/fixtures/restore_reusable_dataset.json delete mode 100644 kive/portal/fixtures/run_api_tests.json delete mode 100644 kive/portal/fixtures/run_component_too_many_checks.json delete mode 100644 kive/portal/fixtures/run_pipelines_recovering_reused_step.json delete mode 100644 kive/portal/fixtures/simple_run.json delete mode 100644 kive/sandbox/execute.py delete mode 100644 kive/sandbox/tests.py delete mode 100644 kive/sandbox/tests_rm.py delete mode 100755 utils/prepare_initial_pipeline.py diff --git a/kive/FixtureFiles/container_run/Datasets/2019_02/example_names.csv b/kive/FixtureFiles/container_run/Datasets/2019_06/example_names.csv similarity index 100% rename from kive/FixtureFiles/container_run/Datasets/2019_02/example_names.csv rename to kive/FixtureFiles/container_run/Datasets/2019_06/example_names.csv diff --git a/kive/archive/management/__init__.py b/kive/archive/management/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/kive/archive/management/commands/__init__.py b/kive/archive/management/commands/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/kive/archive/management/commands/check_complete_runs.py b/kive/archive/management/commands/check_complete_runs.py deleted file mode 100644 index a274fd739..000000000 --- a/kive/archive/management/commands/check_complete_runs.py +++ /dev/null @@ -1,198 +0,0 @@ -from __future__ import print_function -from collections import Counter -from datetime import datetime -import re -from operator import itemgetter - -from django.core.management import call_command -from django.core.management.base import BaseCommand -from django.core.urlresolvers import reverse, resolve -from django.db import connection -from django.db.models import Count -from rest_framework.test import APIRequestFactory, force_authenticate - -from archive.models import Run, RunStep -from metadata.models import kive_user, CompoundDatatype -from librarian.models import ExecRecord, Dataset -from librarian.views import dataset_view -from kive.tests import DuckRequest -from pipeline.models import Pipeline -import itertools - - -class Command(BaseCommand): - help = "Exercise the Run.is_complete() method for performance testing. " - - def handle(self, *args, **options): - self.count_queries(self.test_purge_synch) - - def check_many_runs(self): - run_count = 100 - runs = Run.objects.order_by('-id')[:run_count] - complete_count = sum(1 for run in runs if run.is_complete()) - print('Found {} complete in {} most recent runs.'.format(complete_count, - run_count)) - - def test_dataset_view(self): - dataset_id = self.find_big_dataset().id - request = DuckRequest() - return dataset_view(request, dataset_id) - - def test_dataset_rows(self): - dataset = self.find_big_dataset() - return len(list(dataset.rows(data_check=True, limit=7))) - - def test_many_runsteps(self): - count = 100 - run_steps = list(RunStep.objects.filter( - reused=False).prefetch_related('RSICs').order_by('-id')[:count]) - success_count = sum(1 - for step in run_steps - if step.is_successful()) - print('Found {} successful in {} most recent runs.'.format(success_count, - len(run_steps))) - - def test_purge_synch(self): - call_command('purge', synch=True) - - def test_many_execrecords(self): - count = 100 - execrecords = list(ExecRecord.objects.order_by('-id')[:count]) - fail_count = sum(1 for r in execrecords if r.has_ever_failed()) - print('Found {} failed in {} most recent runs.'.format(fail_count, - len(execrecords))) - - def find_big_dataset(self): - compound_datatypes = CompoundDatatype.objects.annotate( - Count('members')).order_by('-members__count') - for compound_datatype in compound_datatypes: - datasets = Dataset.objects.filter( - structure__compounddatatype=compound_datatype).exclude( - dataset_file='').order_by('-structure__num_rows') - for dataset in datasets: - return dataset - raise RuntimeError('No structured datasets found.') - - def count_queries(self, task): - """ Count the queries triggered by task, and print a summary. - - @param task: a callable that will trigger some database queries. - """ - start_count = len(connection.queries) - start_time = datetime.now() - result = task() - duration = datetime.now() - start_time - end_count = len(connection.queries) - print('{!r} after {} queries and {}.'.format( - result, - end_count - start_count, - duration)) - active_queries = connection.queries[start_count:] - min_time = duration.total_seconds() * 0.01 - slow_queries = [query - for query in active_queries - if float(query['time']) > min_time] - if slow_queries: - print('') - total_slow_time = sum(map(float, map(itemgetter('time'), slow_queries))) - total_time = sum(map(float, map(itemgetter('time'), active_queries))) - print("Slow queries ({:.2f}s for slow and {:.2f}s for all):".format( - total_slow_time, - total_time)) - max_display = 2000 - tail_size = 200 - for query in slow_queries: - display = str(query) - if len(display) > max_display: - display = (display[:max_display-tail_size] + - '...' + - display[-tail_size:]) - print(display) - - table_counts = Counter() - table_times = Counter() - for query in active_queries: - m = re.match('SELECT +"([^"]*)"', query['sql']) - if m: - table_counts[m.group(1)] += 1 - table_times[m.group(1)] += float(query['time']) - if m.group(1) == 'transformation_xputstructureXXX': - print(query['sql']) - print('') - print('Query counts:') - for table, count in table_counts.most_common(20): - print('{}: {}'.format(table, count)) - print('') - print('Query times:') - for table, time in table_times.most_common(20): - print('{}: {}'.format(table, time)) - - return result - - def test_prefetch_new(self, use_transformation_output=False): - queryset = Pipeline.objects.prefetch_related( - 'steps__transformation__outputs__structure').filter(family_id=1)[:25] - pipeline = queryset.first() - # pipelines[0].steps.all()[0].outputs[0].structure - step = pipeline.steps.all()[0] - if use_transformation_output: - transformation = step.transformation - output = transformation.outputs.all()[0] - else: - output = step.outputs[0] - structure = output.structure - return len([structure]) - - def test_prefetch(self): - queryset = Pipeline.objects.prefetch_related( - 'steps__transformation__method__family', - 'steps__transformation__pipeline__family', - 'steps__transformation__method__inputs__structure__compounddatatype__members__datatype', - 'steps__transformation__method__outputs__structure__compounddatatype__members__datatype', - 'steps__transformation__outputs__structure', - 'steps__transformation__method__family', - 'steps__cables_in__custom_wires', - 'steps__cables_in__dest__transformationinput', - 'steps__cables_in__dest__transformationoutput', - 'steps__cables_in__source__transformationinput', - 'steps__cables_in__source__transformationoutput', - 'steps__outputs_to_delete', - 'inputs__structure', - 'inputs__transformation', - 'outcables__source__structure', - 'outcables__source__transformationinput', - 'outcables__source__transformationoutput', - 'outcables__custom_wires__source_pin', - 'outcables__custom_wires__dest_pin', - 'outcables__pipeline', - 'outcables__output_cdt', - 'outputs__structure').filter(family_id=1)[:25] - pipelines = list(queryset) - # pipelines[0].steps.all()[0].outputs[0].structure - steps = list(itertools.chain(*(p.steps.all() for p in pipelines))) - outputs = list(itertools.chain(*(s.outputs for s in steps))) - structures = [o.structure for o in outputs if hasattr(o, 'structure')] - return len(structures) - - def test_ajax(self): - factory = APIRequestFactory() - path = '/api/datasets/' - view, _, _ = resolve(path) - request = factory.get( - path + '?is_granted=true&filters%5B0%5D%5Bkey%5D=uploaded' - '&filters%5B1%5D%5Bkey%5D=cdt&filters%5B1%5D%5Bval%5D=31' - '&page_size=8&page=1') - force_authenticate(request, user=kive_user()) - response = view(request).render() - data = response.render().data - return data['count'] - - def test_ajax_download(self): - factory = APIRequestFactory() - dataset_path = reverse('dataset-download', kwargs={'pk': 283134}) - view, _, _ = resolve(dataset_path) - request = factory.get(dataset_path) - force_authenticate(request, user=kive_user()) - response = view(request, pk=283134) - content = response.content - return content diff --git a/kive/archive/management/commands/checkrcconsistency.py b/kive/archive/management/commands/checkrcconsistency.py deleted file mode 100644 index a3887c53d..000000000 --- a/kive/archive/management/commands/checkrcconsistency.py +++ /dev/null @@ -1,39 +0,0 @@ -from django.core.management.base import BaseCommand -from archive.models import RunStep, RunCable -from optparse import make_option -from itertools import chain - - -class Command(BaseCommand): - help = " Checks for and optionally fixes inconsistencies between the normalized fields " \ - "_successful and _complete on RunSteps and RunCables. " - - def add_arguments(self, parser): - parser.add_argument( - "-f", - "--fix-inconsistencies", - dest="fix_inconsistencies", - choices=['0', '1'], - default='1' - ) - - def handle(self, *args, **options): - fix_inconsistencies = options['fix_inconsistencies'] - - # We only use the _completed/_successful flags on runsteps and runcables. - # TODO: Filter out redacted steps once that field has been added - for rc in chain(RunStep.objects.all(), RunCable.objects.all()): - is_mcompl, is_compl = (rc.is_complete(use_cache=True, dont_save=True), - rc.is_complete(dont_save=not fix_inconsistencies)) - is_msuccs, is_succs = (rc.is_successful(use_cache=True, dont_save=True), - rc.is_successful(dont_save=not fix_inconsistencies)) - - if is_compl != is_mcompl: - self.stdout.write('%s %s (id: %d) is_complete has mark(%s) but was computed to be (%s) ' % ( - type(rc).__name__, rc, rc.id, is_compl, is_mcompl)) - - if is_succs != is_msuccs: - self.stdout.write('%s %s (id: %d) is_successful has mark(%s) but was computed to be (%s) ' % ( - type(rc).__name__, rc, rc.id, is_succs, is_msuccs)) - - self.stdout.write('Completed check!') diff --git a/kive/archive/management/commands/force_reruns.py b/kive/archive/management/commands/force_reruns.py deleted file mode 100644 index ba94b510b..000000000 --- a/kive/archive/management/commands/force_reruns.py +++ /dev/null @@ -1,115 +0,0 @@ -import logging -from time import sleep - -from collections import defaultdict -from django.core.management.base import BaseCommand - -from archive.models import Run, RunBatch -from constants import runstates -from pipeline.models import Pipeline - - -logger = logging.getLogger(__name__) - - -class Command(BaseCommand): - help = "Purges output datasets from runs and requests reruns." - - def add_arguments(self, parser): - parser.add_argument( - "--run_ids", - help='comma-separated list of runs to purge and rerun') - parser.add_argument( - "--pipeline_ids", - help='comma-separated list of pipeline ids to search for runs') - parser.add_argument( - "--skip_pipeline_ids", - help='comma-separated list of pipeline ids to avoid') - parser.add_argument( - "--runs_per_pipeline", - type=int, - default=10, - help='number of runs to purge for each pipeline, chosen randomly') - parser.add_argument( - "--active_count", - type=int, - default=50, - help='number of runs active at one time') - parser.add_argument( - "--outputs", - default="", - help='comma-separated list of output names to purge') - - def handle(self, *args, **options): - # See also: utils/request_reruns.py - logger.info('Starting.') - if options['run_ids']: - run_ids = map(int, options['run_ids'].split(',')) - runs = Run.objects.filter(id__in=run_ids) - else: - if options['pipeline_ids']: - pipeline_ids = map(int, options['pipeline_ids'].split(',')) - else: - pipelines = Pipeline.objects.order_by('id') - if options['skip_pipeline_ids']: - skip_ids = map(int, options['skip_pipeline_ids'].split(',')) - pipelines = pipelines.exclude(id__in=skip_ids) - pipeline_ids = (row['id'] for row in pipelines.values('id')) - run_count = options['runs_per_pipeline'] - runs = self.find_runs(pipeline_ids, run_count) - active_count = options['active_count'] - targets = filter(None, options['outputs'].split(',')) - batch = None - current_pipeline_id = None - for run in runs: - while Run.objects.filter(end_time=None).count() >= active_count: - sleep(10) - - logger.debug('Rerunning %s', run) - for step in run.runsteps_in_order: - execrecord = step.execrecord - if execrecord is not None: - for output in execrecord.execrecordouts_in_order: - output_name = output.generic_output.definite.dataset_name - dataset = output.dataset - if not targets or output_name in targets: - dataset.dataset_file.delete() - if run.pipeline_id != current_pipeline_id: - batch = RunBatch.objects.create(name=str(run.pipeline), - user_id=run.user_id) - current_pipeline_id = run.pipeline_id - rerun = Run.objects.create(pipeline_id=run.pipeline_id, - user_id=run.user_id, - name=run.name, - runbatch=batch) - for old_input in run.inputs.all(): - rerun.inputs.create(dataset=old_input.dataset, - index=old_input.index) - for old_group in run.groups_allowed.all(): - rerun.groups_allowed.add(old_group) - for old_user in run.users_allowed.all(): - rerun.users_allowed.add(old_user) - logger.info('Done.') - - def find_runs(self, pipeline_ids, run_count): - launched_input_ids = defaultdict(set) # {family_id: {input_id}} - for pipeline_id in pipeline_ids: - # We run into trouble when two versions of a pipeline run at the - # same time and try to restore the same dataset at the same time. - pipeline = Pipeline.objects.get(id=pipeline_id) - family_input_ids = launched_input_ids[pipeline.family_id] - pipeline_run_count = 0 - for run in Run.objects.filter(pipeline_id=pipeline_id, - _runstate_id=runstates.SUCCESSFUL_PK).order_by('?'): - input_ids = {run_input.dataset_id - for run_input in run.inputs.order_by('index')} - if not input_ids & family_input_ids: - yield run - family_input_ids |= input_ids - pipeline_run_count += 1 - if pipeline_run_count >= run_count: - break - if pipeline_run_count < run_count: - logger.warn('Only found %d runs for pipeline %s.', - pipeline_run_count, - pipeline) diff --git a/kive/archive/management/commands/purge_batch.py b/kive/archive/management/commands/purge_batch.py deleted file mode 100644 index eeacb6bcb..000000000 --- a/kive/archive/management/commands/purge_batch.py +++ /dev/null @@ -1,31 +0,0 @@ -from django.core.management.base import BaseCommand - -from archive.models import RunBatch - - -class Command(BaseCommand): - help = "Purges output datasets from a run batch." - - def add_arguments(self, parser): - parser.add_argument( - "batch_id", - type=int) - parser.add_argument( - "--outputs", - default="", - help='comma-separated list of output names to purge') - - def handle(self, *args, **options): - batch = RunBatch.objects.get(id=options['batch_id']) - targets = filter(None, options['outputs'].split(',')) - for run in batch.runs.all(): - print(run) - for step in run.runsteps_in_order: - print(" {}".format(step)) - execrecord = step.execrecord - for output in execrecord.execrecordouts_in_order: - output_name = output.generic_output.definite.dataset_name - dataset = output.dataset - if not targets or output_name in targets: - dataset.dataset_file.delete() - print(" {}".format(output_name)) diff --git a/kive/archive/management/commands/set_old_state_fields.py b/kive/archive/management/commands/set_old_state_fields.py deleted file mode 100644 index a9542b59c..000000000 --- a/kive/archive/management/commands/set_old_state_fields.py +++ /dev/null @@ -1,42 +0,0 @@ -from django.core.management.base import BaseCommand - -from archive.models import Run, RunStep, RunOutputCable, RunSIC - - -class Command(BaseCommand): - help = "Sets the _complete and _successful fields for all Runs and RunComponents " \ - "(for migrations beyond v0.7)" - - def handle(self, *args, **options): - - # Ideally, this handles everything through propagating downward to - # its components. - for run in Run.objects.filter(_complete__isnull=True, parent_runstep__isnull=True): - run.is_complete(use_cache=True) - for run in Run.objects.filter(_successful__isnull=True, parent_runstep__isnull=True): - run.is_successful(use_cache=True) - - # Now check any runs that we missed in case their parent run somehow had - # their _complete and _successful flags set. - for run in Run.objects.filter(_complete__isnull=True, parent_runstep__isnull=False): - run.is_complete(use_cache=True) - for run in Run.objects.filter(_successful__isnull=True, parent_runstep__isnull=False): - run.is_successful(use_cache=True) - - # Any steps missed by the above are handled here. - for rs in RunStep.objects.filter(_complete__isnull=True): - rs.is_complete(use_cache=True) - for rs in RunStep.objects.filter(_successful__isnull=True): - rs.is_successful(use_cache=True) - - # Any output cables missed by the above are handled here. - for roc in RunOutputCable.objects.filter(_complete__isnull=True): - roc.is_complete(use_cache=True) - for roc in RunOutputCable.objects.filter(_successful__isnull=True): - roc.is_successful(use_cache=True) - - # Finally, handle any input cables missed by the above. - for rsic in RunSIC.objects.filter(_complete__isnull=True): - rsic.is_complete(use_cache=True) - for rsic in RunSIC.objects.filter(_successful__isnull=True): - rsic.is_successful(use_cache=True) \ No newline at end of file diff --git a/kive/archive/migrations/0108_drop_cables.py b/kive/archive/migrations/0108_drop_cables.py new file mode 100644 index 000000000..6485254f6 --- /dev/null +++ b/kive/archive/migrations/0108_drop_cables.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.21 on 2019-06-14 22:37 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('librarian', '0112_remove_dataset_file_source'), + ('datachecking', '0106_baddata_file_not_stable'), + ('archive', '0107_runinput_ordering_priority_help_text_20170403_1547'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='runoutputcable', + unique_together=set([]), + ), + migrations.RemoveField( + model_name='runoutputcable', + name='pipelineoutputcable', + ), + migrations.RemoveField( + model_name='runoutputcable', + name='run', + ), + migrations.RemoveField( + model_name='runoutputcable', + name='runcomponent_ptr', + ), + migrations.AlterUniqueTogether( + name='runsic', + unique_together=set([]), + ), + migrations.RemoveField( + model_name='runsic', + name='PSIC', + ), + migrations.RemoveField( + model_name='runsic', + name='dest_runstep', + ), + migrations.RemoveField( + model_name='runsic', + name='runcomponent_ptr', + ), + migrations.DeleteModel( + name='RunOutputCable', + ), + migrations.DeleteModel( + name='RunSIC', + ), + ] diff --git a/kive/archive/models.py b/kive/archive/models.py index 6a43f9e21..cfc08320b 100644 --- a/kive/archive/models.py +++ b/kive/archive/models.py @@ -262,59 +262,7 @@ def outcables_in_order(self): return self.runoutputcables.order_by("pipelineoutputcable__output_idx") def clean(self): - """ - Checks coherence of the Run (possibly in an incomplete state). - - The procedure: - - check coherence of start_time and end_time - - if parent_runstep is not None, then pipeline should be - consistent with it - - RunSteps should all be clean, and should be consecutively - numbered starting from 1 - - all associated RunStepOutputCables must be from RunSteps - which are associated (ie. at least in progress), and must - be clean - """ - Run.validate_permissions(self) - # # Access to this Run must not exceed that of the pipeline or of the batch. - # self.validate_restrict_access([self.pipeline]) - # if self.runbatch is not None: - # self.validate_restrict_access([self.runbatch]) - # - # for rtp_input in self.inputs.all(): - # rtp_input.clean() - - if hasattr(self, "not_enough_CPUs"): - self.not_enough_CPUs.clean() - - # # If this is not a top-level run it must have the same access as the top-level run. - # my_top_level_run = self.top_level_run - # if self != my_top_level_run: - # self.validate_identical_access(my_top_level_run) - - # Check that start- and end-time are coherent. - stopwatch.models.Stopwatch.clean(self) - - if self.is_subrun() and self.pipeline != self.parent_runstep.pipelinestep.transformation.definite: - raise ValidationError('Pipeline of Run "{}" is not consistent with its parent RunStep'.format(self)) - - # Go through whatever steps are registered. All must be clean. - for i, runstep in enumerate(self.runsteps.order_by("pipelinestep__step_num"), start=1): - if runstep.pipelinestep.step_num != i: - raise ValidationError('RunSteps of Run "{}" are not consecutively numbered starting from 1' - .format(self)) - runstep.clean() - - # Can't have RunOutputCables from non-existent RunSteps. - # TODO: Should this go in RunOutputCable.clean() ? - for run_outcable in self.runoutputcables.all(): - source_step = run_outcable.pipelineoutputcable.source_step - try: - self.runsteps.get(pipelinestep__step_num=source_step) - except RunStep.DoesNotExist: - raise ValidationError('Run "{}" has a RunOutputCable from step {}, but no corresponding RunStep' - .format(self, source_step)) - run_outcable.clean() + pass @staticmethod def validate_permissions(run): @@ -871,26 +819,7 @@ def increase_permissions_from_json(self, permissions_json): ds.increase_permissions_from_json(permissions_json) def get_all_atomic_runcomponents(self): - """ - Returns an iterable of all atomic RunComponents that belong to this Run. - - This includes RunComponents that belong to sub-Runs. - """ - # This will be a list of querysets that we will use itertools.chain to join. - rc_querysets = [] - - atomic_step_pks = [] - for rs in self.runsteps.all(): - rc_querysets.append(rs.RSICs.all()) - if rs.has_subrun(): - sub_rcs = rs.child_run.get_all_atomic_runcomponents() - rc_querysets.append(sub_rcs) - else: - atomic_step_pks.append(rs.pk) - rc_querysets.append(RunStep.objects.filter(pk__in=atomic_step_pks)) - rc_querysets.append(self.runoutputcables.all()) - - return itertools.chain(*rc_querysets) + pass def eligible_permissions(self, include_runbatch=True): """ @@ -945,70 +874,7 @@ def eligible_permissions(self, include_runbatch=True): return addable_users, addable_groups def cancel_components(self, except_steps=None, except_incables=None, except_outcables=None): - """ - Cancel all components (and sub-Runs) except for those specified. - - Returns True if all components are complete; False otherwise. - - PRE: anything that the fleet is currently running is specified among - the exceptions. - """ - except_steps = except_steps or [] - except_incables = except_incables or [] - except_outcables = except_outcables or [] - - skip_step_pks = (x.pk for x in except_steps if x.parent_run == self) - skip_incable_pks = (x.pk for x in except_incables if x.parent_run == self) - skip_outcable_pks = (x.pk for x in except_outcables if x.parent_run == self) - - # In addition to the steps and outcables, we also need to check that all - # incables are complete. - incables_still_running = False - for step in self.runsteps.exclude(pk__in=skip_step_pks).exclude( - _runcomponentstate__pk__in=runcomponentstates.COMPLETE_STATE_PKS - ): - for rsic in step.RSICs.exclude(pk__in=skip_incable_pks).exclude( - _runcomponentstate__pk__in=runcomponentstates.COMPLETE_STATE_PKS - ): - rsic.cancel(save=True) - - if not step.has_subrun(): - step.cancel(save=True) - - else: - if step.child_run.is_pending(): - step.child_run.start() # transition: Pending->Running - - if step.child_run.is_running(): - step.child_run.cancel() # transition: Running->Cancelling - - all_stopped = step.child_run.cancel_components(except_steps=except_steps, - except_incables=except_incables, - except_outcables=except_outcables) - if all_stopped: - step.child_run.stop(save=True) # transition: Cancelling->Cancelled or Failing->Failed - step.cancel(save=True) - - for outcable in self.runoutputcables.exclude(pk__in=skip_outcable_pks).exclude( - _runcomponentstate__pk__in=runcomponentstates.COMPLETE_STATE_PKS - ): - outcable.cancel(save=True) - - if RunSIC.objects.filter(dest_runstep__in=self.runsteps.all()).exclude( - _runcomponentstate__pk__in=runcomponentstates.COMPLETE_STATE_PKS - ): - incables_still_running = True - - # Return True if everything is complete and False otherwise. - return not ( - incables_still_running or - self.runsteps.exclude( - _runcomponentstate__pk__in=runcomponentstates.COMPLETE_STATE_PKS - ).exists() or - self.runoutputcables.exclude( - _runcomponentstate__pk__in=runcomponentstates.COMPLETE_STATE_PKS - ).exists() - ) + pass class RunInput(models.Model): @@ -1292,25 +1158,13 @@ def top_level_run(self): return self.definite.top_level_run def is_step(self): - try: - self.runstep - except RunStep.DoesNotExist: - return False return True def is_incable(self): - try: - self.runsic - except RunSIC.DoesNotExist: - return False - return True + return False def is_outcable(self): - try: - self.runoutputcable - except RunOutputCable.DoesNotExist: - return False - return True + return False def is_cable(self): return self.is_incable() or self.is_outcable() @@ -1980,36 +1834,7 @@ def get_coordinates(self): return run_coords + (self.pipelinestep.step_num,) def find_compatible_ERs(self, input_datasets): - """ - Find all ExecRecords that are compatible with this RunStep. - - Exclude redacted ones. Permissions of old run must include all - permissions of new run. - @param input_datasets: a list of datasets that have - already been processed by the input cables. To be compatible, an - ExecRecord must have the same inputs in the same order. - @return: generator of ExecRecords - """ - transformation = self.transformation - assert transformation.is_method() - if transformation.definite.reusable == Method.NON_REUSABLE: - return - - execrecord_ids = RunStep.objects.filter( - pipelinestep__transformation=transformation).values('execrecord_id') - query = ExecRecord.objects.filter(id__in=execrecord_ids) - for dataset_idx, dataset in enumerate(input_datasets, 1): - query = query.filter( - execrecordins__generic_input__transformationinput__dataset_idx=dataset_idx, - execrecordins__dataset=dataset) - - new_run = self.top_level_run - for execrecord in query.all(): - if not execrecord.is_redacted(): - extra_users, extra_groups = new_run.extra_users_groups( - [execrecord.generating_run]) - if not(extra_users or extra_groups): - yield execrecord + pass @transaction.atomic def check_ER_usable(self, execrecord): @@ -2091,607 +1916,6 @@ def _first_ER_h(execrecord_summary_list): return list_sorted[0][0], list_sorted[0][1] -class RunCable(RunComponent): - """ - Class inherited by RunSIC and RunOutputCable. - - Since those classes share so much functionality, this - abstract class will encapsulate that stuff and RSIC/ROC - can extend it where necessary. - """ - # Implicit from RunComponent: start_time, end_time, execrecord, - # reused, log, invoked_logs, outputs. - - class Meta: - abstract = True - - @classmethod - def create(cls, cable, parent_record): - if cable.is_incable(): - runcable = RunSIC.create(cable, parent_record) - else: - runcable = RunOutputCable.create(cable, parent_record) - return runcable - - def has_data(self): - """True if associated output exists; False if not.""" - return self.outputs.exists() - - def keeps_output(self): - """ - True if this RunCable retains its output; false otherwise. - - This is an abstract function that must be implemented by - RunSIC and RunOutputCable - """ - pass - - def _cable_type_str(self): - """ - Helper to retrieve the class name of this cable. - - That is, a RunSIC will return "RunSIC" and a - RunOutputCable will return "RunOutputCable". - """ - return self.__class__.__name__ - - def _pipeline_cable(self): - """ - Abstract function that retrieves the PSIC/POC. - """ - pass - - @property - def component(self): - return self.PSIC - - def is_trivial(self): - return self.component.is_trivial() - - def _clean_not_reused(self): - """ - Check coherence of a RunCable which has decided not to reuse an - ExecRecord: - - - if reused is False: - - - call RunComponent._clean_reused - - - if the cable is trivial, there should be no associated Dataset - - otherwise, make sure there is at most one Dataset, and clean it - if it exists - - This is a helper for clean(). - - PRE - This RunCable has reused = False (has decided not to reuse an - ExecRecord). - - OUTPUT - True if there are more checks to do within clean, False if clean - should return right away. - """ - if not RunComponent._clean_not_reused(self): - return False - - # From here on, the ExecLog is known to be complete. - - # If this cable is trivial, there should be no data - # associated. - if self.has_data() and self.outputs.first().has_data(): - if self._pipeline_cable().is_trivial(): - raise ValidationError( - '{} "{}" is trivial and should not have generated any Datasets'.format( - self._cable_type_str(), self) - ) - - # Otherwise, check that there is at most one Dataset - # attached, and clean it. - if self.outputs.count() > 1: - raise ValidationError('{} "{}" should generate at most one Dataset'.format( - self._cable_type_str(), self)) - self.outputs.first().clean() - return True - - def _clean_cable_coherent(self): - """ - Checks that the cable is coherent with its parent. - - This is an abstract function that must be implemented - by both RunSIC and RunOutputCable. - """ - pass - - def _clean_with_execlog(self): - """ - If this RunCable has an ExecLog (that is, code was run during its - execution), make sure it is in a coherent state: - - - if this RunCable does not keep its output or its output is - missing, there should be no existent data associated. - - - This is a helper function for clean. - - PRE - This RunCable has an ExecLog. - """ - # If output of the cable not marked as kept, there shouldn't be a Dataset. - if not self.keeps_output(): - # Check if the attached output has real data associated to it. - if self.has_data() and self.outputs.first().has_data(): - raise ValidationError( - '{} "{}" does not keep its output but a dataset was registered'.format( - self._cable_type_str(), self) - ) - - # If EL shows missing output, there shouldn't be a Dataset. - elif self.log.missing_outputs(): - if self.has_data() and self.outputs.first().has_data(): - raise ValidationError('{} "{}" had missing output but a dataset was registered'.format( - self._cable_type_str(), self)) - - def _clean_without_execlog_reused_check_output(self): - """ - Check coherence of a reused RunCable with no ExecLog. - - In this state, it should not have any registered output Datasets. - - This is a helper for clean(). - - PRE: this RunCable is reused, has no ExecLog, and passes clean - up to the point that this function is invoked. - """ - # Case 1: Completely recycled ER (reused = true): it should - # not have any registered dataset) - if self.outputs.exists(): - raise ValidationError('{} "{}" was reused but has a registered dataset'.format( - self._cable_type_str(), self - )) - - def _clean_without_execlog_not_reused(self): - """ - Check coherence of a non-reused RunCable without an ExecLog. - - In this state, execution is incomplete, so it should not have - any outputs or an ExecRecord. - - This is a helper for clean(). - - PRE: this RunCable is not reused, has no ExecLog, and passes - clean up to the point that this function is invoked. - """ - general_error = '{} "{}" not reused and has no ExecLog'.format(self._cable_type_str(), self) - if self.outputs.exists(): - raise ValidationError("{}, but has a Dataset output".format(general_error)) - if self.execrecord is not None: - raise ValidationError("{}, but has an ExecRecord".format(general_error)) - - def _clean_execrecord(self): - """ - Check coherence of the RunCable's associated ExecRecord. - - This is an abstract function that must be overridden by - RunSIC and RunOutputCable, as most of this is case-specific. - - If the output of the cable is kept and either the record is reused or - it isn't reused and no missing outputs are noted: - - the corresponding ERO should have existent data associated - - - if the PSIC/POC is not trivial and this RunCable does not reuse an ER, - then there should be existent data associated and it should also - be associated to the corresponding ERO. - - - it must represent a PipelineCable - - - The cable is compatible with self.execrecord.general_transf() - - PRE - This RunCable has an ExecRecord. - """ - self.execrecord.complete_clean() - - # If output of the cable is kept and either the record is reused or - # it isn't reused and no missing outputs are noted, - # the corresponding ERO should have existent data. - if self.keeps_output(): - if self.reused or len(self.log.missing_outputs()) == 0: - - # TODO: helper to get the ExecRecordOut without calling first(). - corresp_ero = self.execrecord.execrecordouts.first() - if not corresp_ero.has_data(): - raise ValidationError('{} "{}" keeps its output; ExecRecordOut "{}" should reference existent ' - 'data'.format(self._cable_type_str(), self, corresp_ero)) - - # If reused == False and the cable is not trivial, - # there should be associated data, and it should match that - # of corresp_ero. - if not self.reused and not self._pipeline_cable().is_trivial(): - if not (self.has_data() and self.outputs.first().has_data()): - raise ValidationError('{} "{}" was not reused, trivial, or deleted; it should have ' - 'produced data'.format(self._cable_type_str(), self)) - - if corresp_ero.dataset != self.outputs.first(): - raise ValidationError('Dataset "{}" was produced by {} "{}" but is not in an ERO of ' - 'ExecRecord "{}"'.format(self.outputs.first(), - self._cable_type_str(), - self, - self.execrecord)) - - # June 9, 2014: since PSICs are now allowed to use ERs of POCs and vice versa, the functionality - # that was previously in RunSIC and RunOutputCable._clean_execrecord can now be folded into here. - if not self.execrecord.general_transf().is_cable(): - raise ValidationError('ExecRecord of {} "{}" does not represent a PipelineCable'.format( - self.__class__.__name__, self)) - - elif not self.component.is_compatible(self.execrecord.general_transf()): - raise ValidationError('{} of {} "{}" is incompatible with the cable of its ExecRecord'.format( - self.component.__class__.__name__, self.__class__.__name__, self)) - - def clean(self): - """ - Check coherence of this RunCable. - - In sequence, the checks we perform: - - check coherence of start_time and end_time - - PSIC/POC belongs to dest_runstep.pipelinestep/run.pipeline - - - if an ExecLog is attached, clean it - - - perform relevant coherence checks based on whether the RunCable - has decided to reuse an ExecRecord (see _clean_undecided_reused, - _clean_reused, and _clean_not_reused) - - (from here on execrecord is assumed to be set) - - - clean the ExecRecord(see _clean_execrecord) - - - perform relevant coherence checks based on whether the RunCable - has an ExecLog (ie. code was run) (see _clean_with_execlog - and _clean_without_execlog) - """ - self.logger.debug("Initiating") - - RunComponent.clean(self) - # Check coherence of the times. - stopwatch.models.Stopwatch.clean(self) - - self._clean_cable_coherent() - - self._clean_execlogs() - - if self.reused is None: - self._clean_undecided_reused() - elif self.reused: - self._clean_reused() - elif not self._clean_not_reused(): - return - - self.logger.debug("Checking {}'s ExecLog".format(self._cable_type_str())) - - # Handle cases where the log either exists or does not exist. - if not self.has_log(): - if self.reused: - self._clean_without_execlog_reused_check_output() - else: - self._clean_without_execlog_not_reused() - # We know we don't have an ER at this point so we stop. - return - else: - self._clean_with_execlog() - - # At this point, we know that the log either exists or should - # not exist (i.e. this is a reused cable). - - # If there is no execrecord defined but there is a log, then - # check for spurious CCLs and ICLs and stop. - if self.execrecord is None: - if self.has_log(): - self._clean_has_execlog_no_execrecord_yet() - return - - # Now, we know there to be an ExecRecord. - self._clean_execrecord() - - # Attempt to clean the input_integrity_check. - try: - self.input_integrity_check.clean() - except IntegrityCheckLog.DoesNotExist: - pass - - def find_compatible_ERs(self, input_SD): - """ - Find ExecRecords which may be used by this RunCable. - - INPUTS - input_SD Dataset to feed the cable - - OUTPUTS - list of ExecRecords that are compatible with this cable and input (may be empty). - """ - return self.component.find_compatible_ERs(input_SD, self) - - @transaction.atomic - def check_ER_usable(self, execrecord): - """ - Check that the specified ExecRecord is reusable (fully or not) or unsuccessful. - """ - summary = {"fully reusable": False, "successful": True} - - output_SD = execrecord.execrecordouts.first().dataset - - # Terminal case 1: the found ExecRecord has failed some initial checks. In this case, - # we just return and the RunCable fails. - if not output_SD.usable_in_run(): - self.logger.debug("The ExecRecord ({}) found has a bad output.".format(execrecord)) - summary["successful"] = False - - # Terminal case 2: the ExecRecord passed its checks and provides the output we need. - elif output_SD.is_OK() and (not self.keeps_output() or output_SD.has_data()): - self.logger.debug("Can fully reuse ER {}".format(execrecord)) - summary["fully reusable"] = True - - return summary - - @transaction.atomic - def get_suitable_ER(self, input_SD, reuse_failures=True): - """ - Retrieve a suitable ExecRecord for this RunCable. - - If any of them are failed, we find the failed one with the most outputs, with - ties broken by the smallest PK. - If any of them are fully reusable, we find the fully reusable one satisfying the same criteria. - Otherwise we find whichever one satisfies the same criteria. - - Return a tuple containing the ExecRecord along with its summary (as - produced by check_ER_usable), or None if no appropriate ExecRecord is found. - """ - execrecords = self.find_compatible_ERs(input_SD) - failed = [] - fully_reusable = [] - other = [] - execrecords_sorted = sorted(execrecords, key=attrgetter("pk")) - for er in execrecords_sorted: - curr_summary = self.check_ER_usable(er) - if not curr_summary["successful"]: - failed.append((er, curr_summary)) - elif curr_summary["fully reusable"]: - fully_reusable.append((er, curr_summary)) - else: - other.append((er, curr_summary)) - - if reuse_failures and len(failed) > 0: - return _first_ER_h(failed) - - if len(fully_reusable) > 0: - return _first_ER_h(fully_reusable) - - if len(other) > 0: - return _first_ER_h(other) - - return None, None - - -class RunSIC(RunCable): - """ - Annotates the action of a PipelineStepInputCable within a RunStep. - - Related to :model:`archive.models.RunStep` - Related to :model:`librarian.models.ExecRecord` - Related to :model:`pipeline.models.PipelineStepInputCable` - """ - dest_runstep = models.ForeignKey(RunStep, related_name="RSICs") - PSIC = models.ForeignKey("pipeline.PipelineStepInputCable", related_name="psic_instances") - - # Implicit from RunCable: execrecord, reused, log, output, invoked_logs, start_time, end_time. - - class Meta: - # Uniqueness constraint ensures that no POC is multiply-represented - # within a run step. - unique_together = ("dest_runstep", "PSIC") - - @classmethod - def create(cls, PSIC, runstep): - runsic = cls(PSIC=PSIC, dest_runstep=runstep) - runsic.start() - runsic.clean() - runsic.save() - return runsic - - def _pipeline_cable(self): - """ - Retrieves the PSIC of this RunSIC. - """ - return self.PSIC - - @property - def component(self): - return self.PSIC - - @property - def parent_run(self): - return self.dest_runstep.run - - @property - def top_level_run(self): - """ - Returns the top-level Run this belongs to. - """ - return self.dest_runstep.top_level_run - - @property - def pipeline(self): - return self.PSIC.pipelinestep.pipeline - - @property - def parent(self): - return self.dest_runstep - - def is_step(self): - return False - - def is_incable(self): - return True - - def is_outcable(self): - return False - - # TODO: fix for sub-pipelines - def output_name(self): - return "run{}_step{}_input{}".format(self.parent_run.pk, self.dest_runstep.step_num, self.PSIC.dest.dataset_idx) - - def output_description(self): - run = self.top_level_run - desc = ('Generated data from a run of pipeline "{}" started at {} by {}\n' - .format(self.pipeline, run.start_time, run.user)) - desc += "run: {}\n".format(run.pk) - desc += "user: {}\n".format(run.user) - desc += "step: {}\n".format(self.dest_runstep.step_num) - desc += "input: {}".format(self.PSIC.dest.dataset_name) - return desc - - def keeps_output(self): - """ - True if the underlying PSIC retains its output; False otherwise. - """ - if self.PSIC.is_trivial(): - return False - return self.PSIC.keep_output - - def _clean_cable_coherent(self): - """ - Checks that the PSIC and PipelineStep are coherent. - """ - if (not self.dest_runstep.pipelinestep.cables_in.filter(pk=self.PSIC.pk).exists()): - raise ValidationError('PSIC "{}" does not belong to PipelineStep "{}"' - .format(self.PSIC, self.dest_runstep.pipelinestep)) - - def get_coordinates(self): - """ - Retrieves a tuple of pipeline coordinates of this RunSIC. - - This is simply the coordinates of the RunStep it belongs to. - Implicitly, if you are comparing a RunSIC with a RunStep that has - the same coordinates, the RunSIC is deemed to come first. - """ - return self.dest_runstep.get_coordinates() - - -class RunOutputCable(RunCable): - """ - Annotates the action of a PipelineOutputCable within a run. - - Related to :model:`archive.models.Run` - Related to :model:`librarian.models.ExecRecord` - Related to :model:`pipeline.models.PipelineOutputCable` - """ - run = models.ForeignKey(Run, related_name="runoutputcables") - pipelineoutputcable = models.ForeignKey("pipeline.PipelineOutputCable", related_name="poc_instances") - - # Implicit from RunCable: execrecord, reused, log, output, invoked_logs, start_time, end_time. - - class Meta: - # Uniqueness constraint ensures that no POC is - # multiply-represented within a run. - unique_together = ("run", "pipelineoutputcable") - - @classmethod - def create(cls, pipelineoutputcable, run): - runoutputcable = cls(pipelineoutputcable=pipelineoutputcable, run=run) - runoutputcable.start() - runoutputcable.clean() - runoutputcable.save() - return runoutputcable - - def __str__(self): - return 'RunOutputCable("{}")'.format( - self.pipelineoutputcable.output_name) - - def _pipeline_cable(self): - """ - Retrieves the POC of this RunOutputCable. - """ - return self.pipelineoutputcable - - @property - def component(self): - return self.pipelineoutputcable - - @property - def parent_run(self): - return self.run - - @property - def top_level_run(self): - """ - Returns the top-level Run this belongs to. - """ - return self.run.top_level_run - - @property - def pipeline(self): - return self.pipelineoutputcable.pipeline - - @property - def parent(self): - return self.run - - def is_step(self): - return False - - def is_incable(self): - return False - - def is_outcable(self): - return True - - # TODO: fix for sub-pipelines - def output_name(self): - return "run{}_output{}".format(self.run.pk, self.pipelineoutputcable.output_idx) - - def output_description(self): - run = self.top_level_run - desc = ('Generated data from a run of pipeline "{}" started at {} by {}\n' - .format(self.pipeline, run.start_time, run.user)) - desc += "run: {}\n".format(run.pk) - desc += "user: {}\n".format(run.user) - desc += "output: {}".format(self.pipelineoutputcable.output_name) - return desc - - def keeps_output(self): - """ - True if the underlying POC retains its output; False otherwise. - """ - if self.pipelineoutputcable.is_trivial(): - return False - - if self.run.parent_runstep is None: - return True - - # At this point we know that this is a sub-Pipeline. Check - # if the parent PipelineStep deletes this output. - return not self.run.parent_runstep.pipelinestep.outputs_to_delete.filter( - dataset_idx=self.pipelineoutputcable.output_idx).exists() - - def _clean_cable_coherent(self): - """ - Checks that the POC and Pipeline are coherent. - """ - if not self.run.pipeline.outcables.filter(pk=self.pipelineoutputcable.pk).exists(): - raise ValidationError('POC "{}" does not belong to Pipeline "{}"' - .format(self.pipelineoutputcable, self.run.pipeline)) - - def get_coordinates(self): - """ - Retrieves a tuple of pipeline coordinates of this RunOutputCable. - - This is simply the coordinates of the Run that contains it; - implicitly, any ROCs with these coordinates is deemed to come - after all of the RunSteps belonging to the same Run. - """ - return self.run.get_coordinates() - - def get_upload_path(instance, filename): """ Helper method for uploading dataset_files for Dataset. diff --git a/kive/archive/tests.py b/kive/archive/tests.py deleted file mode 100644 index 281ea6474..000000000 --- a/kive/archive/tests.py +++ /dev/null @@ -1,4425 +0,0 @@ -""" -Kive archive application unit tests. -""" - -import os -import re -import tempfile -import json - -from mock import call, patch, Mock - -from django.contrib.auth.models import User, Group -from django.core.exceptions import ValidationError -from django.core.files import File -from django.core.files.base import ContentFile -from django.utils import timezone -from django.test import TestCase, skipIfDBFeature -from django.core.urlresolvers import reverse, resolve -from django_mock_queries.query import MockSet -from django_mock_queries.mocks import mocked_relations -from rest_framework import status -from rest_framework.test import force_authenticate - -from archive.models import ExecLog, MethodOutput, Run, RunComponent,\ - RunOutputCable, RunStep, RunSIC, RunBatch, RunState -from datachecking.models import BadData -from file_access_utils import compute_md5 -from librarian.models import ExecRecord, Dataset, DatasetStructure - -from kive.tests import BaseTestCases, install_fixture_files, remove_fixture_files -from method.models import Method, MethodFamily, CodeResource -from pipeline.models import Pipeline, PipelineStep, PipelineFamily - -from fleet.workers import Manager -import kive.testing_utils as tools - -# Rather than define everyone_group here, we import this function to prevent compile-time -# database access. -from metadata.models import kive_user, everyone_group, CompoundDatatype - -from constants import groups, runstates - - -class ArchiveTestCaseHelpers(object): - def __init__(self): - pass - - def make_complete_non_reused(self, record, input_SDs, output_SDs): - """ - Helper function to do everything necessary to make a RunStep, - RunOutputCable, or RunStepInputCable complete, when it has not - reused an ExecRecord (ie. make a new ExecRecord). - - """ - if not record.has_started(): - record.start(save=False) - self.make_execlog_and_mark_non_reused_runcomponent(record) - - execrecord = ExecRecord.create(record.log, record.component, input_SDs, output_SDs) - record.execrecord = execrecord - record.finish_successfully(save=True) - - def make_execlog_and_mark_non_reused_runcomponent(self, record): - """Attaches a good ExecLog to a RunComponent.""" - record.reused = False - record.save() - - execlog = ExecLog(record=record, invoking_record=record, start_time=timezone.now(), end_time=timezone.now()) - execlog.save() - if record.is_step(): - MethodOutput(execlog=execlog, return_code=0).save() - - def make_complete_reused(self, record, input_SDs, output_SDs, other_parent): - """ - Helper function to do everything necessary to make a RunStep, - RunOutputCable, or RunStepInputCable complete, when it _has_ - reused an ExecRecord (ie. make an ExecRecord for it to reuse). - """ - if not record.has_started(): - record.start(save=False) - record_type = record.__class__.__name__ - - new_record = record.__class__.create(record.component, other_parent) # this start()s it - - execlog = ExecLog(record=new_record, invoking_record=new_record, start_time=timezone.now(), - end_time=timezone.now()) - execlog.save() - if record_type == "RunStep": - MethodOutput(execlog=execlog, return_code=0).save() - execrecord = ExecRecord.create(execlog, record.component, input_SDs, output_SDs) - - record.execrecord = execrecord - record.reused = True - record.finish_successfully(save=True) - - def complete_RSICs(self, runstep, input_SDs, output_SDs): - """ - Helper function to create and complete all the RunSIC's needed for - a given RunStep. input_SDs and output_SDs are lists of the input and - output datasets for each cable, in order. - """ - for i, cable in enumerate(runstep.pipelinestep.cables_in.order_by("dest__dataset_idx")): - rsic = cable.psic_instances.create(dest_runstep=runstep) - self.make_complete_non_reused(rsic, [input_SDs[i]], [output_SDs[i]]) - - def step_through_runstep_creation(self, bp): - """ - Helper function to step through creation of a RunStep, breaking - at a certain point (see the code for what these points are). - """ - if bp == "empty_runs": - return - - self.step_E1_RS = self.step_E1.pipelinestep_instances.create(run=self.pE_run) - self.step_E1_RS.start(save=True) - if bp == "first_runstep": - return - - self.E03_11_RSIC = self.E03_11.psic_instances.create(dest_runstep=self.step_E1_RS) - self.make_complete_non_reused(self.E03_11_RSIC, [self.raw_dataset], [self.raw_dataset]) - # self.raw_dataset.integrity_checks.create(execlog=self.E03_11_RSIC.log, user=self.myUser) - if bp == "first_rsic": - return - - self.make_complete_non_reused(self.step_E1_RS, [self.raw_dataset], [self.doublet_dataset]) - step1_in_ccl = self.doublet_dataset.content_checks.first() - step1_in_ccl.execlog = self.step_E1_RS.log - step1_in_ccl.save() - self.doublet_dataset.file_source = self.step_E1_RS - self.doublet_dataset.save() - if bp == "first_runstep_complete": - return - - self.step_E2_RS = self.step_E2.pipelinestep_instances.create(run=self.pE_run) - self.step_E2_RS.start(save=True) - if bp == "second_runstep": - return - - self.complete_RSICs(self.step_E2_RS, - [self.triplet_dataset, self.singlet_dataset], - [self.D1_in_dataset, self.singlet_dataset]) - self.E01_21_RSIC = self.step_E2_RS.RSICs.filter(PSIC=self.E01_21).first() - self.E02_22_RSIC = self.step_E2_RS.RSICs.filter(PSIC=self.E02_22).first() - - D1_in_ccl = self.D1_in_dataset.content_checks.first() - D1_in_ccl.execlog = self.E01_21_RSIC.log - D1_in_ccl.save() - - self.singlet_dataset.integrity_checks.create(execlog=self.E02_22_RSIC.log, user=self.myUser) - if bp == "second_runstep_cables_complete": - return - - # Associate and complete sub-Pipeline. - self.pD_run.parent_runstep = self.step_E2_RS - self.pD_run.save() - self.step_D1_RS = self.step_D1.pipelinestep_instances.create(run=self.pD_run) - self.complete_RSICs(self.step_D1_RS, - [self.D1_in_dataset, self.singlet_dataset], - [self.D1_in_dataset, self.singlet_dataset]) - self.D01_11_RSIC = self.step_D1_RS.RSICs.filter(PSIC=self.D01_11).first() - self.D02_12_RSIC = self.step_D1_RS.RSICs.filter(PSIC=self.D02_12).first() - self.D1_in_dataset.integrity_checks.create(execlog=self.D01_11_RSIC.log, user=self.myUser) - self.singlet_dataset.integrity_checks.create(execlog=self.D02_12_RSIC.log, user=self.myUser) - - self.make_complete_non_reused(self.step_D1_RS, [self.D1_in_dataset, self.singlet_dataset], [self.C1_in_dataset]) - C1_ccl = self.C1_in_dataset.content_checks.first() - C1_ccl.execlog = self.step_D1_RS.log - C1_ccl.save() - self.C1_in_dataset.file_source = self.step_D1_RS - self.C1_in_dataset.save() - - pD_ROC = self.pD.outcables.first().poc_instances.create(run=self.pD_run) - self.make_complete_non_reused(pD_ROC, [self.C1_in_dataset], [self.C1_in_dataset]) - self.C1_in_dataset.integrity_checks.create(execlog=pD_ROC.log, user=self.myUser) - - self.step_E2_RS.finish_successfully(save=True) - if bp == "sub_pipeline": - return - - def step_through_run_creation(self, bp): - """ - Helper function to step through creation of a Run. bp is a - breakpoint - these are defined throughout (see the code). - """ - if not hasattr(self, 'myUser'): - self.myUser = User.objects.get(username='john') - self.mB = Method.objects.get(revision_name="mB_name") - self.pD = Pipeline.objects.get(revision_name="pD_name") - self.pE = Pipeline.objects.get(revision_name="pE_name") - self.B1_in = self.mB.inputs.get(dataset_name="B1_in") - self.B2_in = self.mB.inputs.get(dataset_name="B2_in") - self.D1_in = self.pD.inputs.get(dataset_name="D1_in") - self.D2_in = self.pD.inputs.get(dataset_name="D2_in") - self.step_D1 = PipelineStep.objects.get( - pipeline=self.pD, - step_num=1) - self.step_E1 = PipelineStep.objects.get( - pipeline=self.pE, - step_num=1) - self.step_E2 = PipelineStep.objects.get( - pipeline=self.pE, - step_num=2) - self.step_E3 = PipelineStep.objects.get( - pipeline=self.pE, - step_num=3) - self.pE_run = Run.objects.get(pipeline=self.pE, - name='pE_run') - self.raw_dataset = Dataset.objects.get(name='raw_DS') - self.singlet_dataset = Dataset.objects.get( - dataset_file__endswith='singlet_cdt_large.csv') - self.doublet_dataset = Dataset.objects.get(name='doublet') - self.triplet_dataset = Dataset.objects.get( - dataset_file__endswith='step_0_triplet.csv') - self.C1_in_dataset = Dataset.objects.get(name='C1_in_triplet') - self.C1_out_dataset = Dataset.objects.get( - dataset_file__endswith='step_0_singlet.csv') - self.C2_in_dataset = Dataset.objects.get( - dataset_file__endswith='E11_32_output.csv') - self.C2_out_dataset = Dataset.objects.get(name='C2_out') - self.C3_out_dataset = Dataset.objects.get(name='C3_out') - self.E1_out_dataset = Dataset.objects.get(name='E1_out') - self.D1_in_dataset = Dataset.objects.get( - structure__compounddatatype=self.doublet_dataset.structure.compounddatatype, - structure__num_rows=10) - self.D01_11 = self.step_D1.cables_in.get(dest__dataset_idx=1) - self.D02_12 = self.step_D1.cables_in.get(dest__dataset_idx=2) - self.E01_21 = self.step_E2.cables_in.get(dest__dataset_idx=1) - self.E02_22 = self.step_E2.cables_in.get(dest__dataset_idx=2) - self.E21_31 = self.step_E3.cables_in.get(dest__dataset_idx=1) - self.E11_32 = self.step_E3.cables_in.get(dest__dataset_idx=2) - - self.pE_run.start(save=True) - - # Changed May 14, 2014 to add CCLs/ICLs where appropriate. - # Empty Runs. - self.pD_run = self.pD.pipeline_instances.create(user=self.myUser) - self.pD_run.grant_everyone_access() - if bp == "empty_runs": - return - - # First RunStep associated. - self.step_E1_RS = self.step_E1.pipelinestep_instances.create(run=self.pE_run) - self.step_E1_RS.start() - if bp == "first_step": - return - - # First RunSIC associated and completed. - step_E1_RSIC = self.step_E1.cables_in.first().psic_instances.create(dest_runstep=self.step_E1_RS) - if bp == "first_cable_created": - return - - self.make_complete_non_reused(step_E1_RSIC, [self.raw_dataset], [self.raw_dataset]) - icl = self.raw_dataset.integrity_checks.create(execlog=step_E1_RSIC.log, user=self.myUser) - icl.start(save=False) - icl.stop(save=False) - icl.save() - if bp == "first_cable": - return - - # First RunStep completed. - self.make_complete_non_reused(self.step_E1_RS, [self.raw_dataset], [self.doublet_dataset]) - step1_in_ccl = self.doublet_dataset.content_checks.first() - step1_in_ccl.execlog = self.step_E1_RS.log - step1_in_ccl.save() - self.doublet_dataset.file_source = self.step_E1_RS - self.doublet_dataset.save() - if bp == "first_step_complete": - return - - # Second RunStep associated. - self.step_E2_RS = self.step_E2.pipelinestep_instances.create(run=self.pE_run) - if bp == "second_step": - return - - # Sub-pipeline for step 2 - reset step_E2_RS. - # self.step_E2_RS.delete() - # self.step_E2_RS = self.step_E2.pipelinestep_instances.create(run=self.pE_run, reused=None) - self.step_E2_RS.reused = None - self.step_E2_RS.start(save=True) - self.complete_RSICs(self.step_E2_RS, [self.triplet_dataset, self.singlet_dataset], - [self.D1_in_dataset, self.singlet_dataset]) - - self.E01_21_RSIC = self.step_E2_RS.RSICs.filter(PSIC=self.E01_21).first() - self.E02_22_RSIC = self.step_E2_RS.RSICs.filter(PSIC=self.E02_22).first() - - D1_in_ccl = self.D1_in_dataset.content_checks.first() - D1_in_ccl.execlog = self.E01_21_RSIC.log - D1_in_ccl.save() - - icl = self.singlet_dataset.integrity_checks.create(execlog=self.E02_22_RSIC.log, user=self.myUser) - icl.start(save=False) - icl.stop(save=False) - icl.save() - - self.pD_run.parent_runstep = self.step_E2_RS - self.pD_run.start(save=True) - if bp == "sub_pipeline": - return - - # Complete sub-Pipeline. - self.step_D1_RS = self.step_D1.pipelinestep_instances.create(run=self.pD_run) - self.step_D1_RS.start() - self.complete_RSICs(self.step_D1_RS, [self.D1_in_dataset, self.singlet_dataset], - [self.D1_in_dataset, self.singlet_dataset]) - - self.D01_11_RSIC = self.step_D1_RS.RSICs.filter(PSIC=self.D01_11).first() - self.D02_12_RSIC = self.step_D1_RS.RSICs.filter(PSIC=self.D02_12).first() - icl = self.D1_in_dataset.integrity_checks.create(execlog=self.D01_11_RSIC.log, user=self.myUser) - icl.start(save=False) - icl.stop(save=False) - icl.save() - icl = self.singlet_dataset.integrity_checks.create(execlog=self.D02_12_RSIC.log, user=self.myUser) - icl.start(save=False) - icl.stop(save=False) - icl.save() - - self.make_complete_non_reused(self.step_D1_RS, [self.D1_in_dataset, self.singlet_dataset], [self.C1_in_dataset]) - C1_ccl = self.C1_in_dataset.content_checks.first() - C1_ccl.execlog = self.step_D1_RS.log - C1_ccl.save() - self.C1_in_dataset.file_source = self.step_D1_RS - self.C1_in_dataset.save() - - pD_ROC = self.pD.outcables.first().poc_instances.create(run=self.pD_run) - self.make_complete_non_reused(pD_ROC, [self.C1_in_dataset], [self.C1_in_dataset]) - icl = self.C1_in_dataset.integrity_checks.create(execlog=pD_ROC.log, user=self.myUser) - icl.start(save=False) - icl.stop(save=False) - icl.save() - - self.pD_run.stop(save=True) - if bp == "sub_pipeline_complete": - return - - # Third RunStep associated. - self.step_E3_RS = self.step_E3.pipelinestep_instances.create(run=self.pE_run) - self.step_E3_RS.start() - if bp == "third_step": - return - - # Third RunStep completed. - self.complete_RSICs(self.step_E3_RS, [self.C1_in_dataset, self.doublet_dataset], - [self.C1_in_dataset, self.C2_in_dataset]) - - self.E21_31_RSIC = self.step_E3_RS.RSICs.filter(PSIC=self.E21_31).first() - self.E11_32_RSIC = self.step_E3_RS.RSICs.filter(PSIC=self.E11_32).first() - icl = self.C1_in_dataset.integrity_checks.create(execlog=self.E21_31_RSIC.log, user=self.myUser) - icl.start(save=False) - icl.stop(save=False) - icl.save() - - # C2_in_dataset was created here so we associate its CCL with cable - # E11_32. - C2_in_ccl = self.C2_in_dataset.content_checks.first() - C2_in_ccl.execlog = self.E11_32_RSIC.log - C2_in_ccl.save() - - if bp == "third_step_cables_done": - return - - step3_outs = [self.C1_out_dataset, self.C2_out_dataset, self.C3_out_dataset] - self.make_complete_non_reused(self.step_E3_RS, [self.C1_in_dataset, self.C2_in_dataset], step3_outs) - # All of these were first created here, so associate the CCL of C1_out_dataset to step_E3_RS. - # The others are raw and don't have CCLs. - C1_out_ccl = self.C1_out_dataset.content_checks.first() - C1_out_ccl.execlog = self.step_E3_RS.log - C1_out_ccl.save() - - if bp == "third_step_complete": - return - - # Outcables associated. - roc1 = self.pE.outcables.get(output_idx=1).poc_instances.create(run=self.pE_run) - self.make_complete_non_reused(roc1, [self.C1_in_dataset], [self.E1_out_dataset]) - # This was first created here, so associate the CCL appropriately. - E1_out_ccl = self.E1_out_dataset.content_checks.first() - E1_out_ccl.execlog = roc1.log - E1_out_ccl.save() - self.E1_out_dataset.file_source = roc1 - self.E1_out_dataset.save() - - if bp == "first_outcable": - return - - roc2 = self.pE.outcables.get(output_idx=2).poc_instances.create(run=self.pE_run) - self.make_complete_non_reused(roc2, [self.C1_out_dataset], [self.C1_out_dataset]) - roc3 = self.pE.outcables.get(output_idx=3).poc_instances.create(run=self.pE_run) - self.make_complete_non_reused(roc3, [self.C3_out_dataset], [self.C3_out_dataset]) - - # roc2 and roc3 are trivial cables, so we associate integrity checks with C1_out_dataset - # and C3_out_dataset. - icl = self.C1_out_dataset.integrity_checks.create(execlog=roc2.log, user=self.myUser) - icl.start(save=False) - icl.stop(save=False) - icl.save() - icl = self.C3_out_dataset.integrity_checks.create(execlog=roc3.log, user=self.myUser) - icl.start(save=False) - icl.stop(save=False) - icl.save() - - self.pE_run.stop(save=True) - - if bp == "outcables_done": - return - - def step_through_runsic_creation(self, bp): - """ - Helper function to step through creating an RSIC, breaking at a - certain point (see the code). - """ - self.step_E3_RS = self.step_E3.pipelinestep_instances.create(run=self.pE_run) - self.step_E3_RS.start(save=True) - if bp == "runstep_started": - return - - self.E11_32_RSIC = self.E11_32.psic_instances.create(dest_runstep=self.step_E3_RS) - self.E11_32_RSIC.start() - if bp == "rsic_started": - return - - self.make_complete_non_reused(self.E11_32_RSIC, [self.doublet_dataset], [self.C2_in_dataset]) - # C2_in_dataset is created by this cable so associate a CCL appropriately. - C2_ccl = self.C2_in_dataset.content_checks.first() - C2_ccl.execlog = self.E11_32_RSIC.log - C2_ccl.save() - if bp == "rsic_completed": - return - - self.E21_31_RSIC = self.E21_31.psic_instances.create(dest_runstep=self.step_E3_RS) - self.make_complete_non_reused(self.E21_31_RSIC, [self.C1_in_dataset], [self.C1_in_dataset]) - # C1_in_dataset is not created by this RSIC, so associate an ICL. - self.C1_in_dataset.integrity_checks.create(execlog=self.E21_31_RSIC.log, user=self.myUser) - self.make_complete_non_reused(self.step_E3_RS, - [self.C1_in_dataset, self.C2_in_dataset], - [self.C1_out_dataset, self.C2_out_dataset, self.C3_out_dataset]) - # Associate the CCL of C1_out_dataset with step_E3_RS. - C1_out_ccl = self.C1_out_dataset.content_checks.first() - C1_out_ccl.execlog = self.step_E3_RS.log - C1_out_ccl.save() - if bp == "runstep_completed": - return - - def step_through_roc_creation(self, bp): - """Break at an intermediate stage of ROC creation.""" - self.E31_42_ROC = self.E31_42.poc_instances.create(run=self.pE_run) - self.E31_42_ROC.start() - self.E21_41_ROC = self.E21_41.poc_instances.create(run=self.pE_run) - self.E21_41_ROC.start() - if bp == "roc_started": - return - - self.make_complete_non_reused(self.E31_42_ROC, [self.singlet_dataset], [self.singlet_dataset]) - if bp == "trivial_roc_completed": - return - - self.make_complete_non_reused(self.E21_41_ROC, [self.C1_in_dataset], [self.doublet_dataset]) - self.doublet_dataset.file_source = self.E21_41_ROC - self.doublet_dataset.save() - if bp == "custom_roc_completed": - return - - self.step_E2_RS = self.step_E2.pipelinestep_instances.create(run=self.pE_run) - self.step_E2_RS.start(save=True) - self.step_E2_RS.save() - self.pD_run = self.pD.pipeline_instances.create(user=self.myUser) - self.pD_run.start(save=False) - self.pD_run.parent_runstep = self.step_E2_RS - self.pD_run.grant_everyone_access() - self.pD_run.save() - self.D11_21_ROC = self.D11_21.poc_instances.create(run=self.pD_run) - self.D11_21_ROC.start(save=False) - # Define some custom wiring for D11_21: swap the first two columns. - pin1, pin2, _ = (m for m in self.triplet_cdt.members.all()) - self.D11_21.custom_wires.create(source_pin=pin1, dest_pin=pin2) - self.D11_21.custom_wires.create(source_pin=pin2, dest_pin=pin1) - if bp == "subrun": - return - - self.make_complete_non_reused(self.D11_21_ROC, [self.C1_in_dataset], [self.C1_in_dataset]) - self.C1_in_dataset.file_source = self.D11_21_ROC - self.C1_in_dataset.save() - self.C1_in_dataset.content_checks.create(execlog=self.D11_21_ROC.log, - start_time=timezone.now(), - end_time=timezone.now(), - user=self.myUser) - self.D11_21_ROC.stop(save=True) - if bp == "subrun_complete": - return - - -@skipIfDBFeature('is_mocked') -class ArchiveTestCase(TestCase, ArchiveTestCaseHelpers): - fixtures = ["archive_test_environment"] - - def setUp(self): - install_fixture_files("archive_test_environment") - tools.load_archive_test_environment(self) - - def tearDown(self): - remove_fixture_files() - - -class RunComponentTests(ArchiveTestCase): - """Tests of functionality shared by all RunComponents.""" - - def test_clean_execlogs_invoked_logs_cleaned(self): - """Test that _clean_execlogs properly calls clean on its invoked logs.""" - self.step_through_run_creation("outcables_done") - - # For every RunComponent invoked during this run, break each of its invoked_logs and see if it appears. - atomicrunsteps = [] - for runstep in RunStep.objects.all(): - if runstep.transformation.is_method(): - atomicrunsteps.append(runstep) - runcomponents = (atomicrunsteps + list(RunSIC.objects.all()) + list(RunOutputCable.objects.all())) - - for runcomponent in runcomponents: - # Skip RunComponents that are not part of this Run. - if runcomponent.top_level_run != self.pE_run: - continue - - for invoked_log in runcomponent.invoked_logs.all(): - original_el_start_time = invoked_log.start_time - invoked_log.start_time = None - invoked_log.save() - self.assertRaisesRegexp( - ValidationError, - 'Stopwatch "{}" does not have a start time but it has an end time'.format(invoked_log), - runcomponent.clean) - invoked_log.start_time = original_el_start_time - invoked_log.save() - - def test_clean_execlogs_invoked_logs_ICLs_CCLs_cleaned(self): - """Test that ICLs and CCLs are appropriately cleaned in _clean_execlogs.""" - self.step_through_run_creation("outcables_done") - - # For every RunComponent invoked during this run, break each of its ICLs/CCLs and see if it appears. - atomicrunsteps = [] - for runstep in RunStep.objects.all(): - if runstep.transformation.is_method(): - atomicrunsteps.append(runstep) - runcomponents = (atomicrunsteps + list(RunSIC.objects.all()) + list(RunOutputCable.objects.all())) - - for runcomponent in runcomponents: - # Skip RunComponents that are not part of this Run. - if runcomponent.top_level_run != self.pE_run: - continue - - for invoked_log in runcomponent.invoked_logs.all(): - for checklog in (list(invoked_log.integrity_checks.all()) + list(invoked_log.content_checks.all())): - original_checklog_start_time = checklog.start_time - checklog.start_time = None - checklog.save() - self.assertRaisesRegexp( - ValidationError, - 'Stopwatch "{}" does not have a start time but it has an end time'.format(checklog), - runcomponent.clean) - checklog.start_time = original_checklog_start_time - checklog.save() - - def test_clean_execlogs_log_not_among_invoked_logs(self): - """A RunComponent's log should be among its invoked_logs if any invoked_logs exist.""" - self.step_through_run_creation("outcables_done") - - # Imagine that step 3 invokes step 1 but not itself. Note that this would break the Run overall - # but we're only looking to check for errors local to a single RunComponent. - step_3_el = self.step_E3_RS.log - step_1_el = self.step_E1_RS.log - - step_1_el.invoking_record = self.step_E3_RS - step_1_el.save() - - step_3_el.invoking_record = self.pE_run.runoutputcables.get(pipelineoutputcable=self.E21_41) - step_3_el.save() - - self.assertRaisesRegexp(ValidationError, - re.escape('ExecLog of {} "{}" is not included with its invoked ExecLogs'.format( - self.step_E3_RS.__class__.__name__, self.step_E3_RS - )), - self.step_E3_RS.clean) - - def test_clean_execlogs_log_set_before_invoked_ExecLogs_complete(self): - """A RunComponent's log should not be set before all invoked_logs are complete.""" - self.step_through_run_creation("third_step_complete") - - # Imagine that step 3 invokes step 1 and itself. Note that this would break the Run overall - # but we're only looking to check for errors local to a single RunComponent. - step_1_el = self.step_E1_RS.log - step_1_el.invoking_record = self.step_E3_RS - step_1_el.save() - - # Make step_1_el incomplete. - step_1_el.end_time = None - step_1_el.save() - - self.assertRaisesRegexp(ValidationError, - re.escape('ExecLog of {} "{}" is set before all invoked ExecLogs are complete'.format( - self.step_E3_RS.__class__.__name__, self.step_E3_RS - )), - self.step_E3_RS.clean) - - # We no longer require that invoked logs finish all their checks before the current one - # runs. FIXME check if this makes sense before removing this test outright. - # def test_clean_execlogs_log_set_before_invoked_ExecLogs_finish_checks(self): - # """A RunComponent's log should not be set before all invoked_logs finish their checks.""" - # self.step_through_run_creation("third_step_complete") - # - # # Imagine that step 3 invokes step 1 and itself. Note that this would break the Run overall - # # but we're only looking to check for errors local to a single RunComponent. - # step_1_el = self.step_E1_RS.log - # step_1_el.invoking_record = self.step_E3_RS - # step_1_el.save() - # - # # Remove step_1_el's ContentCheckLog. - # step_1_el.content_checks.first().delete() - # - # self.assertRaisesRegexp( - # ValidationError, - # re.escape( - # 'Invoked ExecLogs preceding log of {} "{}" did not successfully pass all of their checks'.format( - # self.step_E3_RS.__class__.__name__, self.step_E3_RS - # )), - # self.step_E3_RS.clean) - - def test_clean_execlogs_runcomponent_invokes_previous_runcomponent(self): - """Testing clean on a RunComponent which invoked a previous RunComponent in the correct fashion.""" - self.step_through_run_creation("third_step_complete") - - # Imagine that step 3 invokes step 1 and itself. Note that this would break the Run overall - # but we're only looking to check for errors local to a single RunComponent. - step_1_el = self.step_E1_RS.log - step_1_el.invoking_record = self.step_E3_RS - step_1_el.save() - - self.assertIsNone(self.step_E3_RS.clean()) - - # def test_clean_execlogs_runcomponent_invoked_by_subsequent_runcomponent(self): - # """ - # Testing clean on a RunComponent whose ExecLog was invoked by a subsequent RunComponent. - # """ - # # Run two pipelines, where the second reuses parts from the first. - # # self.run_pipelines_recovering_reused_step() - # - # # The ExecLog of the first RunStep in sandbox_two's run should have been invoked by - # # the transformation of step 2. - # run_two_step_one = self.sandbox_two.run.runsteps.get(pipelinestep__step_num=1) - # run_two_step_two = self.sandbox_two.run.runsteps.get(pipelinestep__step_num=2) - # - # self.assertEquals(run_two_step_one.log.invoking_record.definite, run_two_step_two) - # self.assertIsNone(run_two_step_one.clean()) - # self.assertIsNone(run_two_step_two.clean()) - - def test_clean_undecided_reused_invoked_logs_exist(self): - """A RunComponent that has not decided on reuse should not have any invoked logs.""" - self.step_through_run_creation("third_step_cables_done") - step_one_el = self.step_E1_RS.log - step_one_el.invoking_record = self.step_E3_RS - step_one_el.save() - - self.assertRaisesRegexp( - ValidationError, - re.escape('{} "{}" has not decided whether or not to reuse an ExecRecord' - "; no steps or cables should have been invoked".format("RunStep", self.step_E3_RS)), - self.step_E3_RS.clean - ) - - def test_clean_reused_invoked_logs_exist(self): - """A RunComponent that reuses an ExecRecord should not have any invoked logs.""" - self.step_through_runstep_creation("first_runstep_complete") - self.step_E1_RS.reused = True - for curr_output in self.step_E1_RS.outputs.all(): - curr_output.file_source = None - curr_output.save() - - self.assertRaisesRegexp( - ValidationError, - re.escape('{} "{}" reused an ExecRecord; no steps or cables should have been invoked'.format( - "RunStep", self.step_E1_RS)), - self.step_E1_RS.clean - ) - - def test_clean_not_reused_runcomponent_log_invoked_elsewhere(self): - """Testing clean on a RunComponent whose log was invoked elsewhere by a subsequent RunComponent.""" - self.step_through_run_creation("third_step_complete") - - # Imagine that step 3 invokes step 1 and itself. Note that this would break the Run overall - # but we're only looking to check for errors local to a single RunComponent. - step_1_el = self.step_E1_RS.log - step_1_el.invoking_record = self.step_E3_RS - step_1_el.save() - - self.assertRaisesRegexp( - ValidationError, - re.escape('{} "{}" is not reused and has not completed its own ExecLog but does have an ExecRecord'.format( - "RunStep", self.step_E1_RS - )), - self.step_E1_RS.clean) - - def test_clean_has_execlog_no_execrecord_yet_has_checks(self): - """Testing clean on a RunComponent that has an ExecLog but no ExecRecord yet also has data checks.""" - self.step_through_run_creation("outcables_done") - - for rc in RunComponent.objects.all(): - if rc.definite.top_level_run == self.pE_run: - # Unset its ExecRecord and clean it. - rc.execrecord = None - self.assertRaisesRegexp( - ValidationError, - '{} "{}" does not have an ExecRecord so should not have any data checks'.format( - rc.definite.__class__.__name__, rc.definite - ), - rc.definite.clean()) - - -@skipIfDBFeature('is_mocked') -class RunComponentInvokedBySubsequentTests(TestCase): - fixtures = ["run_pipelines_recovering_reused_step"] - - def setUp(self): - install_fixture_files("run_pipelines_recovering_reused_step") - - def tearDown(self): - remove_fixture_files() - - def test_clean_execlogs_runcomponent_invoked_by_subsequent_runcomponent(self): - """ - Testing clean on a RunComponent whose ExecLog was invoked by a subsequent RunComponent. - """ - # In the fixture, we ran two pipelines, where the second reused parts from the first. - pipeline_two = Pipeline.objects.get(family__name="p_two", revision_name="v1") - run_two = pipeline_two.pipeline_instances.first() - - # The ExecLog of the first RunStep in sandbox_two's run should have been invoked by - # the transformation of step 2. - run_two_step_one = run_two.runsteps.get(pipelinestep__step_num=1) - run_two_step_two = run_two.runsteps.get(pipelinestep__step_num=2) - - self.assertEquals(run_two_step_one.log.invoking_record.definite, run_two_step_two) - self.assertIsNone(run_two_step_one.clean()) - self.assertIsNone(run_two_step_two.clean()) - - -class RunStepTests(ArchiveTestCase): - - def test_RunStep_clean_wrong_pipeline(self): - """ - A RunStep which has a PipelineStep from one Pipeline, but a Run - for a different Pipeline, is not clean. - """ - self.step_through_runstep_creation("empty_runs") - runstep = self.step_D1.pipelinestep_instances.create(run=self.pE_run) - self.assertRaisesRegexp(ValidationError, - re.escape('PipelineStep "{}" of RunStep "{}" does not belong to Pipeline "{}"' - .format(self.step_D1, runstep, self.pE)), - runstep.clean) - - def test_RunStep_clean_child_run_for_method(self): - """ - A RunStep which represents a Method should not have a child_run - defined. - """ - self.step_through_runstep_creation("first_runstep") - self.pD_run.parent_runstep = self.step_E1_RS - self.pD_run.save() - self.assertRaisesRegexp(ValidationError, - re.escape('PipelineStep of RunStep "{}" is not a Pipeline but a child run exists' - .format(self.step_E1_RS)), - self.step_E1_RS.clean) - - def test_RunStep_clean_no_RunSICs(self): - """ - A RunStep with no RunSIC's is clean. - """ - self.step_through_runstep_creation("first_runstep") - self.assertIsNone(self.step_E1_RS.clean()) - - def test_RunStep_clean_incomplete_RunSIC(self): - """ - A RunStep with an incomplete RunSIC is not clean. - """ - self.step_through_runstep_creation("first_runstep") - # Follow through step_through_runstep_creation, stopping short of completing the input cable. - self.E03_11_RSIC = self.E03_11.psic_instances.create(dest_runstep=self.step_E1_RS) - - self.assertRaisesRegexp(ValidationError, - re.escape('{} "{}" is not complete'.format("RunSIC", self.E03_11_RSIC)), - self.step_E1_RS.clean) - - def test_RunStep_complete_RunSIC(self): - """ - A RunStep with a complete RunSIC is clean. - """ - self.step_through_runstep_creation("first_rsic") - self.assertIsNone(self.step_E1_RS.clean()) - - def test_RunStep_inputs_unquenched_with_data(self): - """ - A RunStep with unquenched input cables, but an associated - Dataset, is not clean. - """ - self.step_through_runstep_creation("first_runstep") - self.doublet_dataset.file_source = self.step_E1_RS - self.doublet_dataset.save() - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" inputs not quenched; no data should have been generated' - .format(self.step_E1_RS)), - self.step_E1_RS.clean) - - def test_RunStep_input_unquenched_with_reused(self): - """ - A RunStep with unquenched input cables, but which has decided - not to reuse an ExecRecord, is not clean. - """ - self.step_through_runstep_creation("first_runstep") - self.step_E1_RS.reused = False - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" inputs not quenched; reused and execrecord should not be set' - .format(self.step_E1_RS)), - self.step_E1_RS.clean) - - def test_RunStep_input_unquenched_with_execrecord(self): - """ - A RunStep with unquenched input cables, but which has an - associated ExecRecord, is not clean. - """ - self.step_through_runstep_creation("first_runstep_complete") - self.E03_11_RSIC.delete() - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" inputs not quenched; reused and execrecord should not be set' - .format(self.step_E1_RS)), - self.step_E1_RS.clean) - - def test_RunStep_input_unquenched_with_log(self): - """ - A RunStep with unquenched input cables, but which has an - associated ExecLog, is not clean. - """ - self.step_through_runstep_creation("first_runstep_complete") - self.E03_11_RSIC.delete() - self.step_E1_RS.execrecord = None - self.step_E1_RS.reused = None - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" inputs not quenched; no log should have been generated' - .format(self.step_E1_RS)), - self.step_E1_RS.clean) - - def test_RunStep_input_unquenched_invokes_other_RunComponents(self): - """ - A RunStep with unquenched input cables should not have any invoked_logs. - """ - # This is a broken setup: step 1 can't have been invoked by step 3. - # Still, we're just checking step 3 here. - self.step_through_run_creation("third_step") - - step_one_el = self.step_E1_RS.log - step_one_el.invoking_record = self.step_E3_RS - step_one_el.save() - - self.assertRaisesRegexp( - ValidationError, - re.escape('RunStep "{}" inputs not quenched; no other steps or cables should have been invoked' - .format(self.step_E3_RS)), - self.step_E3_RS.clean) - - def test_RunStep_input_unquenched_with_child_run(self): - """ - A RunStep with unquenched input cables, but which has a - child_run, is not clean. - """ - self.step_through_runstep_creation("second_runstep") - self.pD_run.parent_runstep = self.step_E2_RS - self.pD_run.save() - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" inputs not quenched; child_run should not be set' - .format(self.step_E2_RS)), - self.step_E2_RS.clean) - - def test_RunStep_clean_inputs_quenched(self): - """ - A RunStep with all its inputs quenched is clean. - """ - self.step_through_runstep_creation("second_runstep_cables_complete") - self.assertIsNone(self.step_E2_RS.clean()) - - def test_RunStep_clean_undecided_reused_with_execrecord(self): - """ - A RunStep which has not decided whether to reuse an ExecRecord, - but which has one associated, is not clean. - """ - self.step_through_runstep_creation("first_rsic") - - other_run = self.pE.pipeline_instances.create(user=self.myUser) - other_run.grant_everyone_access() - other_runstep = self.step_E1.pipelinestep_instances.create(run=other_run) - rsic = self.E03_11.psic_instances.create(dest_runstep=other_runstep) - self.make_complete_non_reused(rsic, [self.raw_dataset], [self.raw_dataset]) - self.make_complete_non_reused(other_runstep, [self.raw_dataset], [self.doublet_dataset]) - - self.step_E1_RS.execrecord = other_runstep.execrecord - - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" has not decided whether or not to reuse an ExecRecord; ' - 'execrecord should not be set'.format(self.step_E1_RS)), - self.step_E1_RS.clean) - - def test_RunStep_clean_undecided_reused_with_execlog(self): - """ - A RunStep which has not decided whether to reuse an ExecRecord, - but which has an ExecLog, is not clean. - """ - self.step_through_runstep_creation("first_runstep_complete") - self.step_E1_RS.reused = None - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" has not decided whether or not to reuse an ExecRecord; ' - 'no log should have been generated'.format(self.step_E1_RS)), - self.step_E1_RS.clean) - - def test_RunStep_clean_undecided_reused_with_data(self): - """ - A RunStep which has not decided whether to reuse an ExecRecord, - but which has output Datasets, is not clean. - """ - # Give step_E1_RS a complete ExecLog. - self.step_through_runstep_creation("first_runstep_complete") - - # To bypass the check for quenched inputs, we have to create - # another ExecRecord which matches step_E1. - other_run = self.pE.pipeline_instances.create(user=self.myUser) - other_run.grant_everyone_access() - other_runstep = self.step_E1.pipelinestep_instances.create(run=other_run) - rsic = self.E03_11.psic_instances.create(dest_runstep=other_runstep) - self.make_complete_non_reused(rsic, [self.raw_dataset], [self.raw_dataset]) - self.make_complete_non_reused(other_runstep, [self.raw_dataset], [self.doublet_dataset]) - - self.step_E1_RS.reused = None - self.step_E1_RS.execrecord = other_runstep.execrecord - - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" has not decided whether or not to reuse an ExecRecord; ' - 'no log should have been generated'.format(self.step_E1_RS)), - self.step_E1_RS.clean) - - def test_RunStep_clean_reused_with_data(self): - """ - A RunStep which has decided to reuse an ExecRecord, but which - has output Datasets, is not clean. - """ - self.step_through_runstep_creation("first_runstep_complete") - self.step_E1_RS.reused = True - self.doublet_dataset.file_source = self.step_E1_RS - self.doublet_dataset.save() - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" reused an ExecRecord and should not have generated any Datasets' - .format(self.step_E1_RS)), - self.step_E1_RS.clean) - - def test_RunStep_clean_subpipeline_with_execlog(self): - """ - A RunStep which has a child run should have no ExecLog. - """ - self.step_through_runstep_creation("sub_pipeline") - ExecLog(record=self.step_E2_RS, invoking_record=self.step_E2_RS, - start_time=timezone.now(), end_time=timezone.now()).save() - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" represents a sub-pipeline so no log should be associated' - .format(self.step_E2_RS)), - self.step_E2_RS.clean) - - def test_RunStep_clean_subpipeline_with_dataset(self): - """ - A RunStep which has a child run should have no associated output - Datasets. - """ - self.step_through_runstep_creation("second_runstep_cables_complete") - self.step_E2_RS.outputs.add(self.singlet_dataset) - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" represents a sub-pipeline and should not have generated any ' - 'data'.format(self.step_E2_RS)), - self.step_E2_RS.clean) - - def test_RunStep_clean_subpipeline_with_reused(self): - """ - A RunStep which has a child run should not have set reused. - """ - self.step_through_runstep_creation("second_runstep_cables_complete") - self.step_E2_RS.reused = True - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" represents a sub-pipeline so reused should not be set' - .format(self.step_E2_RS)), - self.step_E2_RS.clean) - - def test_RunStep_clean_subpipeline_with_execrecord(self): - """ - A RunStep which has a child run should not have an execrecord. - """ - self.step_through_runstep_creation("second_runstep_cables_complete") - other_run = self.pE.pipeline_instances.create(user=self.myUser) - other_run.grant_everyone_access() - other_runstep = self.step_E2.pipelinestep_instances.create(run=other_run) - execlog = ExecLog(record=other_runstep, invoking_record=other_runstep, - start_time=timezone.now(), end_time=timezone.now()) - execlog.save() - execrecord = ExecRecord(generator=execlog) - execrecord.save() - self.step_E2_RS.execrecord = execrecord - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" represents a sub-pipeline so execrecord should not be set' - .format(self.step_E2_RS)), - self.step_E2_RS.clean) - - def test_RunStep_clean_subpipeline_good(self): - """ - A RunStep representing a sub-pipeline, which has not set - reused and has no ExecLog, is clean. - """ - self.step_through_runstep_creation("second_runstep") - self.assertIsNone(self.step_E2_RS.clean()) - - def test_RunStep_clean_reused_no_execrecord(self): - """ - A RunStep which has decided to reuse an ExecRecord, but doesn't - have one associated yet, is clean. - """ - # May 14, 2014: fixed this test to reflect the way things work now. - self.step_through_runstep_creation("first_rsic") - - other_run = self.pE.pipeline_instances.create(user=self.myUser) - other_run.grant_everyone_access() - self.make_complete_reused(self.step_E1_RS, [self.raw_dataset], [self.doublet_dataset], other_run) - - self.step_E1_RS.execrecord = None - self.step_E1_RS.outputs.clear() - self.assertIsNone(self.step_E1_RS.clean()) - - def test_RunStep_clean_non_reused_bad_data(self): - """ - A RunStep which has decided not to reuse an ExecRecord, - and has bad output data, is not clean. - """ - self.step_through_runstep_creation("first_runstep_complete") - self.step_E1_RS.reused = False - self.doublet_dataset.file_source = self.step_E1_RS - self.doublet_dataset.MD5_checksum = "foo" - self.doublet_dataset.save() - with open(self.doublet_dataset.dataset_file.path, 'rb') as f: - checksum = compute_md5(f) - - self.assertRaisesRegexp(ValidationError, - re.escape('File integrity of "{}" lost. Current checksum "{}" does not equal expected ' - 'checksum "{}"'.format(self.doublet_dataset, checksum, "foo")), - self.step_E1_RS.clean) - - def test_RunStep_clean_non_reused_good_data(self): - """ - A RunStep which has decided not to reuse an ExecRecord, and has - clean output data, is clean. - """ - self.step_through_runstep_creation("first_runstep_complete") - self.doublet_dataset.MD5_checksum = self.doublet_dataset.compute_md5() - self.doublet_dataset.save() - self.assertIsNone(self.step_E1_RS.clean()) - - def test_RunStep_clean_good_child_run(self): - """ - A RunStep with a child_run which is clean, is also clean. - """ - self.step_through_runstep_creation("sub_pipeline") - self.assertIsNone(self.step_E2_RS.clean()) - - def test_RunStep_clean_bad_child_run(self): - """ - A RunStep with a child_run which is not clean, is also not - clean. Note: this isn't quite the same as the original test, - I'm having trouble hitting the "execrecord should not be set" - error. - """ - self.step_through_runstep_creation("sub_pipeline") - - self.step_D1_RS.reused = None - self.step_D1_RS.save() - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" has not decided whether or not to reuse an ExecRecord; ' - 'no log should have been generated'.format(self.step_D1_RS)), - self.step_E2_RS.clean) - - def test_RunStep_clean_bad_execrecord(self): - """ - A RunStep whose ExecRecord is not clean, is also not clean. - """ - self.step_through_runstep_creation("first_runstep_complete") - execrecord = self.step_E1_RS.execrecord - execrecord.execrecordins.first().delete() - self.assertRaisesRegexp(ValidationError, - re.escape('Input(s) to ExecRecord "{}" are not quenched'.format(execrecord)), - self.step_E1_RS.clean) - - def test_RunStep_clean_good_execrecord(self): - """ - A RunStep representing a Method with a clean ExecRecord and no - other problems, is clean. - """ - self.step_through_runstep_creation("first_runstep_complete") - self.assertIsNone(self.step_E1_RS.clean()) - - def test_RunStep_execrecord_wrong_transformation(self): - """ - If a RunStep has an associated ExecRecord, it must point to - the same transformation that the RunStep does. - """ - self.step_through_runstep_creation("sub_pipeline") - self.step_E1_RS.execrecord = self.step_D1_RS.execrecord - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" points to transformation "{}" but corresponding ExecRecord ' - 'does not'.format(self.step_E1_RS, self.step_E1)), - self.step_E1_RS.clean) - - def test_RunStep_deleted_output_with_data(self): - """ - A RunStep with an output marked for deletion, should not have - any Datasets associated to that output. - """ - self.step_through_runstep_creation("first_step_complete") - self.step_E1.outputs_to_delete.add(self.mA.outputs.get(dataset_name="A1_out")) - self.step_E1.save() - output = self.step_E1.transformation.outputs.first() - self.assertRaisesRegexp(ValidationError, - re.escape('Output "{}" of RunStep "{}" is deleted; no data should be associated' - .format(output, self.step_E1_RS)), - self.step_E1_RS.clean) - - def test_RunStep_missing_output_with_data(self): - """ - A RunStep with a missing output, should not have any Datasets - associated to that output. - """ - self.step_through_runstep_creation("first_step_complete") - - content_check = self.step_E1_RS.log.content_checks.first() - BadData(contentchecklog=content_check, missing_output=True).save() - - output = self.step_E1.transformation.outputs.first() - self.assertRaisesRegexp(ValidationError, - re.escape('Output "{}" of RunStep "{}" is missing; no data should be associated' - .format(output, self.step_E1_RS)), - self.step_E1_RS.clean) - - def test_RunStep_output_not_in_ExecRecord(self): - """ - A RunStep with a Dataset not in its ExecRecord is not clean. - """ - self.step_through_runstep_creation("first_step") - self.triplet_dataset.file_source = self.step_E1_RS - self.triplet_dataset.save() - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" generated Dataset "{}" but it is not in its ExecRecord' - .format(self.step_E1_RS, self.triplet_dataset)), - self.step_E1_RS.clean) - - def test_RunStep_subpipeline_complete(self): - """ - A RunStep with a complete and clean child run is itself clean - and complete. - """ - self.step_through_runstep_creation("sub_pipeline") - self.assertTrue(self.step_E2_RS.is_complete()) - self.assertIsNone(self.step_E2_RS.complete_clean()) - - def test_RunStep_complete_clean_no_execrecord(self): - """ - A RunStep with no ExecRecord is not complete. - """ - self.step_through_runstep_creation("first_runstep") - self.assertFalse(self.step_E1_RS.is_complete()) - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" is not complete'.format(self.step_E1_RS)), - self.step_E1_RS.complete_clean) - - def test_RunStep_bad_clean_propagation(self): - """ - A RunStep which is not clean, also should not pass - complete_clean. - """ - self.step_through_runstep_creation("first_runstep") - self.doublet_dataset.file_source = self.step_E1_RS - self.doublet_dataset.save() - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" inputs not quenched; no data should have been generated' - .format(self.step_E1_RS)), - self.step_E1_RS.complete_clean) - - def test_RunStep_keeps_output_true(self): - """ - A RunStep with retained output. - """ - self.step_through_runstep_creation("first_runstep") - self.assertTrue(self.step_E1_RS.keeps_output(self.A1_out)) - - def test_RunStep_keeps_output_false(self): - """ - A RunStep with deleted output. - """ - self.step_through_runstep_creation("first_runstep") - self.step_E1.add_deletion(self.A1_out) - self.assertFalse(self.step_E1_RS.keeps_output(self.A1_out)) - - def test_RunStep_keeps_output_multiple_outputs(self): - """ - A RunStep with several outputs, none deleted. - """ - # This is copied from RunTests.step_through_run_creation. - - # Third RunStep associated. - self.step_E3_RS = self.step_E3.pipelinestep_instances.create(run=self.pE_run) - - self.assertTrue(self.step_E3_RS.keeps_output(self.C1_out)) - self.assertTrue(self.step_E3_RS.keeps_output(self.C2_rawout)) - self.assertTrue(self.step_E3_RS.keeps_output(self.C3_rawout)) - - def test_RunStep_keeps_output_multiple_outputs_some_deleted(self): - """ - A RunStep with several outputs, some deleted. - """ - # This is copied from RunTests.step_through_run_creation. - - # Third RunStep associated. - self.step_E3_RS = self.step_E3.pipelinestep_instances.create(run=self.pE_run) - self.step_E3.add_deletion(self.C1_out) - self.step_E3.add_deletion(self.C3_rawout) - - self.assertFalse(self.step_E3_RS.keeps_output(self.C1_out)) - # The deletions shouldn't affect C2_rawout. - self.assertTrue(self.step_E3_RS.keeps_output(self.C2_rawout)) - self.assertFalse(self.step_E3_RS.keeps_output(self.C3_rawout)) - - -@skipIfDBFeature('is_mocked') -class RunComponentTooManyChecks(TestCase): - """ - Tests that check clean() on the case where a RunComponent has too much datachecking. - """ - fixtures = ["run_component_too_many_checks"] - - def setUp(self): - install_fixture_files("run_component_too_many_checks") - self.user_bob = User.objects.get(username="bob") - - def tearDown(self): - remove_fixture_files() - - def test_RunStep_clean_too_many_integrity_checks(self): - """RunStep should have <=1 integrity check for each output.""" - runstep = None - for runstep in RunStep.objects.all(): - if (runstep.execrecord is not None and - runstep.execrecord.execrecordouts.count() > 0 and - runstep.has_log()): - break - log = runstep.log - sd = runstep.execrecord.execrecordouts.first().dataset - log.integrity_checks.create(dataset=sd, user=self.user_bob) - log.integrity_checks.create(dataset=sd, user=self.user_bob) - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" has multiple IntegrityCheckLogs for output ' - 'Dataset {} of ExecLog "{}"'.format(runstep, sd, log)), - runstep.clean) - - def test_RunStep_clean_too_many_integrity_checks_invoked(self): - """Invoked logs of RunStep should have <=1 integrity check for each output.""" - runstep = None - for runstep in RunStep.objects.all(): - if (runstep.execrecord is not None and - runstep.execrecord.execrecordouts.count() > 0 and - runstep.has_log() and - runstep.invoked_logs.count() > 1): - break - for log in runstep.invoked_logs.all(): - sd = log.record.execrecord.execrecordouts.first().dataset - extra_check_1 = log.integrity_checks.create(dataset=sd, user=self.user_bob) - extra_check_2 = log.integrity_checks.create(dataset=sd, user=self.user_bob) - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" has multiple IntegrityCheckLogs for output ' - 'Dataset {} of ExecLog "{}"'.format(runstep, sd, log)), - runstep.clean) - extra_check_1.delete() - extra_check_2.delete() - - def test_RunStep_clean_too_many_content_checks(self): - """RunStep should have <=1 content check for each output.""" - runstep = None - for runstep in RunStep.objects.all(): - if (runstep.execrecord is not None and - runstep.execrecord.execrecordouts.count() > 0 and - runstep.has_log()): - break - log = runstep.log - sd = runstep.execrecord.execrecordouts.first().dataset - log.content_checks.create(dataset=sd, user=self.user_bob) - log.content_checks.create(dataset=sd, user=self.user_bob) - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" has multiple ContentCheckLogs for output ' - 'Dataset {} of ExecLog "{}"'.format(runstep, sd, log)), - runstep.clean) - - def test_RunStep_clean_too_many_content_checks_invoked(self): - """RunStep should have <=1 content check for each output.""" - runstep = None - for runstep in RunStep.objects.all(): - if (runstep.execrecord is not None and - runstep.execrecord.execrecordouts.count() > 0 and - runstep.has_log() and - runstep.invoked_logs.count() > 1): - break - for log in runstep.invoked_logs.all(): - sd = runstep.execrecord.execrecordouts.first().dataset - extra_check_1 = log.content_checks.create(dataset=sd, user=self.user_bob) - extra_check_2 = log.content_checks.create(dataset=sd, user=self.user_bob) - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" has multiple ContentCheckLogs for output ' - 'Dataset {} of ExecLog "{}"'.format(runstep, sd, log)), - runstep.clean) - extra_check_1.delete() - extra_check_2.delete() - - -class RunTests(ArchiveTestCase): - - def test_Run_is_subrun_True(self): - """ - A Run which has a parent RunStep should register as being a subrun. - """ - self.step_through_run_creation("sub_pipeline") - self.assertTrue(self.pD_run.is_subrun()) - - def test_Run_is_subrun_False(self): - """ - A Run which has no parent RunStep should not be a subrun. - """ - self.step_through_run_creation("empty_runs") - self.assertFalse(self.pE_run.is_subrun()) - - def test_Run_clean_not_started(self): - """ - A Run which has been created, but nothing in it has executed - yet, is clean. - """ - self.step_through_run_creation("empty_runs") - self.assertIsNone(self.pE_run.clean()) - - def test_Run_clean_inconsistent_parent_runstep(self): - """ - A sub-Run whose parent RunStep does not match its Pipeline is - not clean. - """ - self.step_through_run_creation("second_step") - self.pD_run.parent_runstep = self.step_E1_RS - self.assertRaisesRegexp(ValidationError, - re.escape('Pipeline of Run "{}" is not consistent with its parent RunStep' - .format(self.pD_run)), - self.pD_run.clean) - - def test_Run_clean_consistent_parent_runstep(self): - """ - A sub-Run whose parent RunStep matches its Pipeline is clean. - """ - self.step_through_run_creation("sub_pipeline") - self.assertIsNone(self.pD_run.clean()) - - def test_Run_clean_first_runstep_incomplete(self): - """ - A Run whose first RunStep is associated and incomplete, and - nothing else is, is clean. - """ - self.step_through_run_creation("first_step") - self.assertIsNone(self.pE_run.clean()) - - def test_Run_clean_first_runstep_complete(self): - """ - A Run whose first RunStep is associated and complete, and - nothing else is, is clean. - """ - self.step_through_run_creation("first_step_complete") - self.assertIsNone(self.pE_run.clean()) - - def test_Run_clean_second_runstep_incomplete(self): - """ - A Run whose second RunStep is associated and incomplete, and - whose first RunStep is complete, is clean. - """ - self.step_through_run_creation("second_step") - self.assertIsNone(self.pE_run.clean()) - - def test_Run_clean_previous_incomplete_runstep(self): - """ - A Run whose second RunStep is associated, but whose first step - is not complete, is clean. - - TODO: when we add the check to RunStep.clean that the - Datasets feeding it are present, this will fail by - propagation. The test setup will need to be modified to put in - place the inputs to steps 1 and 2. - """ - self.step_through_run_creation("second_step") - self.step_E1_RS.execrecord = None - self.step_E1_RS.reused = None - self.step_E1_RS.log.delete() - self.step_E1_RS.save() - self.assertIsNone(self.pE_run.clean()) - - def test_Run_clean_badly_numbered_steps(self): - """ - A Run with steps not consecutively numbered from 1 to n is not - clean. - """ - self.step_through_run_creation("second_step") - self.step_E1_RS.delete() - self.assertRaisesRegexp(ValidationError, - re.escape('RunSteps of Run "{}" are not consecutively numbered starting from 1' - .format(self.pE_run)), - self.pE_run.clean) - - def test_Run_clean_properly_numbered_steps(self): - """ - A Run with steps consecutively numbered from 1 to n is clean. - """ - self.step_through_run_creation("second_step") - self.assertIsNone(self.pE_run.clean()) - - def test_Run_bad_first_RunStep_propagation(self): - """ - A Run whose first RunStep is not clean, and no other RunSteps - are associated, is not clean. - """ - self.step_through_run_creation("first_step_complete") - self.step_E1_RS.reused = None - self.step_E1_RS.RSICs.first().delete() - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" inputs not quenched; reused and execrecord should not be set' - .format(self.step_E1_RS)), - self.pE_run.clean) - - def test_Run_bad_second_RunStep_propagation(self): - """ - A Run whose first RunStep is clean, but whose second RunStep - is not, is not clean. - """ - self.step_through_run_creation("second_step") - self.step_E2_RS.reused = True - self.step_E2_RS.save() - self.assertRaisesRegexp(ValidationError, - re.escape('RunStep "{}" represents a sub-pipeline so reused should not be set' - .format(self.step_E2_RS)), - self.pE_run.clean) - - def test_Run_clean_RunOutputCable_no_RunStep(self): - """ - A Run with a RunOutputCable from a non-associated RunStep is not - clean. - """ - self.step_through_run_creation("sub_pipeline") - self.E31_42.poc_instances.create(run=self.pE_run) - self.assertRaisesRegexp(ValidationError, - re.escape('Run "{}" has a RunOutputCable from step {}, but no corresponding RunStep' - .format(self.pE_run, 3)), - self.pE_run.clean) - - def test_Run_clean_outcable_incomplete_last_RunStep(self): - """ - A Run with a RunOutputCable, and a final RunStep which is clean - but not complete, is clean. - """ - self.step_through_run_creation("third_step") - self.pE.outcables.first().poc_instances.create(run=self.pE_run) - self.assertIsNone(self.pE_run.clean()) - - def test_Run_clean_two_complete_RunSteps(self): - """ - A three step Run with two complete RunSteps, but no third - RunStep, is clean. - """ - self.step_through_run_creation("sub_pipeline") - self.assertIsNone(self.pE_run.clean()) - - def test_Run_clean_all_RunSteps_complete_no_outcables(self): - """ - A Run which has all its steps complete, but no RunOutputCables, - is clean. - """ - self.step_through_run_creation("third_step_complete") - self.assertIsNone(self.pE_run.clean()) - - def test_Run_clean_bad_RunOutputCable_propagation(self): - """ - A Run with an bad RunOutputCable is not clean (not quite the - same as the old test case, I can't make it work). - """ - self.step_through_run_creation("outcables_done") - cable1 = self.pE_run.runoutputcables.first() - cable1.reused = None - cable1.save() - self.assertRaisesRegexp(ValidationError, - re.escape('{} "{}" has not decided whether or not to reuse an ExecRecord; ' - 'no log should have been generated'.format("RunOutputCable", cable1)), - self.pE_run.clean) - - def test_Run_clean_one_complete_RunOutputCable(self): - """ - A Run with one good RunOutputCable, and no others, is clean. - """ - self.step_through_run_creation("first_outcable") - self.assertIsNone(self.pE_run.clean()) - - def test_Run_clean_all_complete_RunOutputCables(self): - """ - A Run with all RunOutputCables complete and clean, is clean. - """ - self.step_through_run_creation("outcables_done") - self.assertIsNone(self.pE_run.clean()) - self.assertTrue(self.pE_run.is_complete()) - self.assertIsNone(self.pE_run.complete_clean()) - - def test_Run_clean_permissions_exceed_RunBatch(self): - """A Run's permissions should not exceed those of its RunBatch.""" - rb = RunBatch(user=self.pE_run.user) - rb.save() - self.pE_run.runbatch = rb - self.assertRaisesRegexp( - ValidationError, - re.escape("Group(s) Everyone cannot be granted access'"), - self.pE_run.clean - ) - - def test_Run_clean_permissions_do_not_exceed_RunBatch(self): - """A Run whose permissions don't exceed those of its RunBatch is OK.""" - rb = RunBatch(user=self.pE_run.user) - rb.save() - rb.grant_everyone_access() - self.pE_run.runbatch = rb - # This doesn't raise an exception. - self.pE_run.clean() - - -class RunSICTests(ArchiveTestCase): - - def test_RunSIC_clean_wrong_pipelinestep(self): - """ - A RunSIC whose PipelineStepInputCable does not belong to its - RunStep's PipelineStep, is not clean. - """ - self.step_through_runsic_creation("runstep_started") - rsic = self.E01_21.psic_instances.create(dest_runstep=self.step_E3_RS) - self.assertRaisesRegexp(ValidationError, - re.escape('PSIC "{}" does not belong to PipelineStep "{}"' - .format(self.E01_21, self.step_E3)), - rsic.clean) - - def test_RunSIC_clean_unset_reused(self): - """ - A RunSIC whose PipelineStepInputCable and RunStep are - consistent, but which has not set reused yet, is clean. - """ - self.step_through_runsic_creation("rsic_started") - self.assertIsNone(self.E11_32_RSIC.clean()) - - def test_RunSIC_clean_unset_reused_with_data(self): - """ - A RunSIC which has not decided whether to reuse an ExecRecord, - but which has associated data, is not clean. - """ - self.step_through_runsic_creation("rsic_started") - self.doublet_dataset.file_source = self.E11_32_RSIC - self.doublet_dataset.save() - self.assertRaisesRegexp(ValidationError, - re.escape('RunSIC "{}" has not decided whether or not to reuse an ExecRecord; ' - 'no Datasets should be associated'.format(self.E11_32_RSIC)), - self.E11_32_RSIC.clean) - - def test_RunSIC_clean_unset_reused_with_execrecord(self): - """ - A RunSIC which has not decided whether to reuse an ExecRecord, - but which has one associated, is not clean. - """ - self.step_through_runsic_creation("rsic_started") - self.E11_32_RSIC.reused = None - - other_run = self.pE.pipeline_instances.create(user=self.myUser) - other_run.grant_everyone_access() - other_runstep = self.step_E3.pipelinestep_instances.create(run=other_run) - other_rsic = self.E11_32.psic_instances.create(dest_runstep=other_runstep) - self.make_complete_non_reused(other_rsic, [self.doublet_dataset], [self.C2_in_dataset]) - self.E11_32_RSIC.execrecord = other_rsic.execrecord - - self.assertRaisesRegexp(ValidationError, - re.escape('RunSIC "{}" has not decided whether or not to reuse an ExecRecord; ' - 'execrecord should not be set yet'.format(self.E11_32_RSIC)), - self.E11_32_RSIC.clean) - - def test_RunSIC_clean_reused_with_data(self): - """ - A RunSIC which has reused an existing ExecRecord, but has an - associated Dataset, is not clean. - """ - self.step_through_runsic_creation("rsic_completed") - self.E11_32_RSIC.reused = True - self.doublet_dataset.file_source = self.E11_32_RSIC - self.doublet_dataset.save() - self.assertRaisesRegexp(ValidationError, - re.escape('RunSIC "{}" reused an ExecRecord and should not have generated any Datasets' - .format(self.E11_32_RSIC)), - self.E11_32_RSIC.clean) - - def test_RunSIC_clean_reused_bad_execrecord(self): - """ - A RunSIC whose ExecRecord is not clean, is not itself clean. - """ - self.step_through_runsic_creation("rsic_started") - other_run = self.pE.pipeline_instances.create(user=self.myUser) - other_run.grant_everyone_access() - other_runstep = self.step_E3.pipelinestep_instances.create(run=other_run) - self.make_complete_reused(self.E11_32_RSIC, [self.doublet_dataset], [self.C2_in_dataset], other_runstep) - - ero = self.E11_32_RSIC.execrecord.execrecordouts.first() - self.C1_in.execrecordouts_referencing.add(ero) - self.assertRaisesRegexp( - ValidationError, - re.escape('CDT of Dataset "{}" is not a restriction of the CDT of the fed TransformationInput "{}"' - .format(ero.dataset, ero.generic_output.definite)), - self.E11_32_RSIC.clean) - - def test_RunSIC_clean_reused_incompatible_execrecord(self): - """ - A RunSIC which is reusing an ExecRecord for an incompatible - PipelineStepInputCable is not clean. - """ - self.step_through_runsic_creation("rsic_started") - - # Create an incompatible RunSIC. - runsic = self.E21_31.psic_instances.create(dest_runstep=self.step_E3_RS) - self.make_complete_non_reused(runsic, [self.C1_in_dataset], [self.C1_in_dataset]) - - run = self.pE.pipeline_instances.create(user=self.myUser) - run.grant_everyone_access() - runstep = self.step_E3.pipelinestep_instances.create(run=run) - self.make_complete_reused(self.E11_32_RSIC, [self.doublet_dataset], [self.C2_in_dataset], runstep) - - self.E11_32_RSIC.execrecord = runsic.execrecord - self.assertRaisesRegexp( - ValidationError, - re.escape('PipelineStepInputCable of RunSIC "{}" is incompatible with the cable of its ExecRecord'.format( - self.E11_32_RSIC)), - self.E11_32_RSIC.clean) - - def test_RunSIC_clean_reused_execrecord_wrong_object(self): - """ - A RunSIC's ExecRecord must be for a PipelineStepInputCable and - not some other pipeline component (reused case). - """ - self.step_through_runsic_creation("rsic_started") - other_run = self.pE.pipeline_instances.create(user=self.myUser) - other_run.grant_everyone_access() - other_runstep = self.step_E3.pipelinestep_instances.create(run=other_run) - - self.make_complete_reused(self.E11_32_RSIC, [self.doublet_dataset], [self.C2_in_dataset], other_runstep) - self.E21_31_RSIC = self.E21_31.psic_instances.create(dest_runstep=self.step_E3_RS) - self.make_complete_non_reused(self.E21_31_RSIC, [self.C1_in_dataset], [self.C1_in_dataset]) - self.make_complete_non_reused(self.step_E3_RS, - [self.C1_in_dataset, self.C2_in_dataset], - [self.C1_out_dataset, self.C2_out_dataset, self.C3_out_dataset]) - - self.E11_32_RSIC.execrecord = self.step_E3_RS.execrecord - self.assertRaisesRegexp(ValidationError, - re.escape('ExecRecord of RunSIC "{}" does not represent a PipelineCable' - .format(self.E11_32_RSIC)), - self.E11_32_RSIC.clean) - # Check of propagation: - self.assertRaisesRegexp(ValidationError, - re.escape('ExecRecord of RunSIC "{}" does not represent a PipelineCable' - .format(self.E11_32_RSIC)), - self.E11_32_RSIC.complete_clean) - - def test_RunSIC_clean_reused_psic_keeps_output_no_data(self): - """ - A RunSIC reusing an ExecRecord, whose PipelineStepInputCable - keeps its output, should have data in its ExecRecordOut. - """ - self.step_through_runsic_creation("rsic_started") - run = self.pE.pipeline_instances.create(user=self.myUser) - run.grant_everyone_access() - runstep = self.step_E3.pipelinestep_instances.create(run=run) - self.E11_32.keep_output = True - self.E11_32.save() - self.make_complete_reused(self.E11_32_RSIC, [self.doublet_dataset], [self.C2_in_dataset], runstep) - ero = self.E11_32_RSIC.execrecord.execrecordouts.first() - - self.assertTrue(self.E11_32_RSIC.keeps_output()) - # Removed May 12, 2014: a reused RunSIC has no log. - # self.assertListEqual(self.E11_32_RSIC.log.missing_outputs(), []) - self.assertFalse(ero.has_data()) - self.assertRaisesRegexp(ValidationError, - re.escape('RunSIC "{}" keeps its output; ExecRecordOut "{}" should reference existent ' - 'data'.format(self.E11_32_RSIC, ero)), - self.E11_32_RSIC.clean) - - def test_RunSIC_clean_reused_psic_keeps_output_with_data(self): - """ - A RunSIC reusing an ExecRecord, whose PipelineStepInputCable - keeps its output, should have data in its ExecRecordOut. - """ - self.step_through_runsic_creation("rsic_started") - - # Make another RSIC which is reused by E11_32_RSIC. - other_run = self.pE.pipeline_instances.create(user=self.myUser) - other_run.grant_everyone_access() - other_run.save() - other_RS = self.step_E3.pipelinestep_instances.create(run=other_run) - - self.make_complete_reused(self.E11_32_RSIC, [self.doublet_dataset], [self.C2_in_dataset], other_RS) - - self.assertIsNone(self.E11_32_RSIC.clean()) - - def test_RunSIC_clean_reused_complete_RSIC(self): - """ - A RunSIC reusing an ExecRecord, whose PipelineStepInputCable - keeps its output, having data in its ExecRecordOut, is complete - and clean. - """ - self.step_through_runsic_creation("rsic_started") - other_run = self.pE.pipeline_instances.create(user=self.myUser) - other_run.grant_everyone_access() - other_runstep = self.step_E3.pipelinestep_instances.create(run=other_run) - - self.make_complete_reused(self.E11_32_RSIC, [self.doublet_dataset], [self.E11_32_output_dataset], - other_runstep) - self.E11_32.keep_output = True - - self.assertTrue(self.E11_32_RSIC.is_complete()) - self.assertIsNone(self.E11_32_RSIC.complete_clean()) - - def test_RunSIC_clean_not_reused_no_execrecord(self): - """ - A RunSIC which has decided not to reuse an ExecRecord, but - which doesn't have one yet, is clean. - """ - self.step_through_runsic_creation("rsic_started") - self.E11_32_RSIC.reused = False - self.assertIsNone(self.E11_32_RSIC.clean()) - - def test_RunSIC_clean_not_reused_bad_execrecord(self): - """ - A RunSIC which is not reusing an ExecRecord, but has a bad - ExecRecord associated, is not clean. - """ - self.step_through_runsic_creation("rsic_completed") - self.E11_32_RSIC.reused = False - ero = self.E11_32_RSIC.execrecord.execrecordouts.first() - self.C1_in.execrecordouts_referencing.add(ero) - self.assertRaisesRegexp( - ValidationError, - re.escape( - 'CDT of Dataset "{}" is not a restriction of the ' - 'CDT of the fed TransformationInput "{}"'.format( - ero.dataset, ero.generic_output.definite)), - self.E11_32_RSIC.clean) - - def test_RunSIC_clean_not_reused_incompatible_execrecord(self): - """ - A RunSIC which is not reusing an ExecRecord, and has an - ExecRecord for an incompatible PipelineStepInputCable, is - not clean. - """ - self.step_through_runsic_creation("rsic_complete") - self.E11_32_RSIC.reused = False - - # Create an incompatible RunSIC. - runstep = self.step_E2.pipelinestep_instances.create(run=self.pE_run) - runsic = self.E02_22.psic_instances.create(dest_runstep=runstep) - self.make_complete_non_reused(runsic, [self.singlet_dataset], [self.singlet_dataset]) - self.E11_32_RSIC.execrecord = runsic.execrecord - self.assertRaisesRegexp( - ValidationError, - re.escape('PipelineStepInputCable of RunSIC "{}" is incompatible with the cable of its ExecRecord'.format( - self.E11_32_RSIC)), - self.E11_32_RSIC.clean) - - def test_RunSIC_clean_not_reused_execrecord_wrong_object(self): - """ - A RunSIC's ExecRecord must be for a PipelineStepInputCable and - not some other pipeline component (non-reused case). - """ - self.step_through_runsic_creation("runstep_completed") - self.E11_32_RSIC.reused = False - - self.E11_32_RSIC.execrecord = self.step_E3_RS.execrecord - self.assertRaisesRegexp(ValidationError, - re.escape('ExecRecord of RunSIC "{}" does not represent a PipelineCable' - .format(self.E11_32_RSIC)), - self.E11_32_RSIC.clean) - # Check of propagation: - self.assertRaisesRegexp(ValidationError, - re.escape('ExecRecord of RunSIC "{}" does not represent a PipelineCable' - .format(self.E11_32_RSIC)), - self.E11_32_RSIC.complete_clean) - - def test_RunSIC_clean_not_reused_psic_discards_output_no_data(self): - """ - A RunSIC not reusing an ExecRecord, whose PipelineStepInputCable - does not keep its output, should not have data in its ExecRecordOut. - """ - self.step_through_runsic_creation("rsic_completed") - self.E11_32_RSIC.reused = False - self.assertIsNone(self.E11_32_RSIC.clean()) - - def test_RunSIC_clean_not_reused_psic_discards_output_with_data(self): - """ - A RunSIC not reusing an ExecRecord, whose PipelineStepInputCable - does not keep its output, should not have data in its ExecRecordOut. - """ - self.step_through_runsic_creation("rsic_completed") - self.E11_32_RSIC.reused = False - self.E11_32_RSIC.outputs.add(self.E11_32_output_dataset) - self.assertRaisesRegexp(ValidationError, - re.escape('RunSIC "{}" does not keep its output but a dataset was registered' - .format(self.E11_32_RSIC)), - self.E11_32_RSIC.clean) - - def test_RunSIC_clean_not_reused_psic_keeps_output_no_data(self): - """ - A RunSIC not reusing an ExecRecord, whose PipelineStepInputCable - keeps its output, should have data in its ExecRecordOut. - """ - self.step_through_runsic_creation("rsic_completed") - self.E11_32_RSIC.reused = False - self.E11_32.keep_output = True - self.E11_32_RSIC.save() - ero = self.E11_32_RSIC.execrecord.execrecordouts.first() - self.assertRaisesRegexp( - ValidationError, - re.escape('RunSIC "{}" keeps its output; ExecRecordOut "{}" should reference existent ' - 'data'.format(self.E11_32_RSIC, ero)), - self.E11_32_RSIC.clean - ) - - def test_RunSIC_clean_not_reused_psic_keeps_output_with_data(self): - """ - A RunSIC not reusing an ExecRecord, whose PipelineStepInputCable - keeps its output, should have data in its ExecRecordOut, and it should - also be among its own outputs. - """ - self.step_through_runsic_creation("rsic_completed") - self.E11_32_RSIC.reused = False - self.E11_32.keep_output = True - ero = self.E11_32_RSIC.execrecord.execrecordouts.first() - - fake_data = "x,y\nHello,World" - ero.dataset.dataset_file.save("FakeData.csv", ContentFile(fake_data)) - ero.dataset.MD5_checksum = ero.dataset.compute_md5() - - ero.dataset.MD5_checksum = ero.dataset.compute_md5() - ero.dataset.save() - self.E11_32_RSIC.outputs.add(ero.dataset) - self.assertIsNone(self.E11_32_RSIC.clean()) - - def test_RunSIC_clean_not_reused_nontrivial_no_data(self): - """ - A RunSIC which is not reused, non-trivial, and is for a - PipelineStepInputCable which keeps its output, must have - produced data. - """ - self.step_through_runsic_creation("rsic_completed") - self.E11_32_RSIC.reused = False - self.E11_32.keep_output = True - ero = self.E11_32_RSIC.execrecord.execrecordouts.first() - ero.dataset = self.E11_32_output_dataset - ero.save() - self.assertRaisesRegexp(ValidationError, - re.escape('RunSIC "{}" was not reused, trivial, or deleted; it should have produced ' - 'data'.format(self.E11_32_RSIC)), - self.E11_32_RSIC.clean) - - def test_RunSIC_clean_not_reused_nontrivial_wrong_data(self): - """ - A RunSIC which is nontrivial, is not reusing an ExecRecord, and - is for a PipelineStepInputCable which keeps its output, must - have produced the same Dataset as is recorded in its - ExecRecordOut. - """ - self.step_through_runsic_creation("rsic_completed") - self.E11_32_RSIC.reused = False - self.E11_32.keep_output = True - - # Associate different datasets to RSIC and associated ERO. - # First, the interloper. - self.doublet_dataset.file_source = self.E11_32_RSIC - self.doublet_dataset.save() - - # Swap out the proper output Dataset for one that retains its data, since - # we've made the PSIC keep its output. - ero = self.E11_32_RSIC.execrecord.execrecordouts.first() - orig_ero_dataset = ero.dataset - orig_ero_dataset.file_source = None - orig_ero_dataset.save() - - ero.dataset = self.E11_32_output_dataset - ero.dataset.file_source = None - ero.dataset.save() - ero.save() - - self.E11_32_output_dataset = ero.dataset - self.E11_32_output_dataset.save() - - self.assertRaisesRegexp(ValidationError, - re.escape('Dataset "{}" was produced by RunSIC "{}" but is not in an ERO of ' - 'ExecRecord "{}"'.format(self.doublet_dataset, self.E11_32_RSIC, - self.E11_32_RSIC.execrecord)), - self.E11_32_RSIC.clean) - - #### - # keeps_output tests added March 26, 2014 -- RL. - def test_RunSIC_keeps_output_trivial(self): - """ - A trivial RunSIC should have keeps_output() return False regardless of the keep_output setting. - """ - self.step_through_runsic_creation("runstep_completed") - self.E21_31.keep_output = True - self.assertFalse(self.E11_32_RSIC.keeps_output()) - self.E21_31.keep_output = False - self.assertFalse(self.E11_32_RSIC.keeps_output()) - - def test_RunSIC_keeps_output_true(self): - """ - A RunSIC that keeps its output should have keeps_output() return True. - """ - self.step_through_runsic_creation("rsic_completed") - self.E11_32.keep_output = True - self.assertTrue(self.E11_32_RSIC.keeps_output()) - - def test_RunSIC_keeps_output_false(self): - """ - A RunSIC that discards its output should have keeps_output() return False. - """ - self.step_through_runsic_creation("rsic_completed") - self.E11_32.keep_output = False - self.assertFalse(self.E11_32_RSIC.keeps_output()) - - -class RunOutputCableTests(ArchiveTestCase): - - def test_ROC_clean_correct_parent_run(self): - """PipelineOutputCable belongs to parent Run's Pipeline. - - A RunOutputCable's PipelineOutputCable must belong to the - Pipeline of its parent Run. - """ - self.step_through_roc_creation("roc_started") - self.assertIsNone(self.E31_42_ROC.clean()) - - def test_ROC_clean_wrong_parent_run(self): - """PipelineOutputCable is for Pipeline not of parent Run. - - A RunOutputCable's PipelineOutputCable must belong to the - Pipeline of its parent Run. - """ - self.step_through_roc_creation("roc_started") - pD_run = self.pD.pipeline_instances.create(user=self.myUser) - self.E31_42_ROC.run = pD_run - self.assertRaisesRegexp(ValidationError, - re.escape('POC "{}" does not belong to Pipeline "{}"'.format(self.E31_42, self.pD)), - self.E31_42_ROC.clean) - - def test_ROC_clean_unset_reused_with_data(self): - """Reused is not set but data is associated. - - A RunOutputCable which has not decided whether to reuse an - ExecRecord should not have generated any data. - """ - self.step_through_roc_creation("roc_started") - self.C1_out_dataset.file_source = self.E31_42_ROC - self.C1_out_dataset.save() - self.assertRaisesRegexp(ValidationError, - re.escape('RunOutputCable "{}" has not decided whether or not to reuse an ExecRecord; ' - 'no Datasets should be associated'.format(self.E31_42_ROC)), - self.E31_42_ROC.clean) - - def test_ROC_clean_unset_reused_with_execrecord(self): - """Reused is not set but an ExecRecord is associated. - - A RunOutputCable which has not decided whether to reuse an - ExecRecord should not have one associated. - """ - self.step_through_roc_creation("roc_started") - - # Create a compatible ExecRecord to associate. - other_run = self.pE.pipeline_instances.create(user=self.myUser) - other_run.grant_everyone_access() - other_roc = self.E31_42.poc_instances.create(run=other_run) - self.make_complete_non_reused(other_roc, [self.C1_out_dataset], [self.C1_out_dataset]) - self.E31_42_ROC.execrecord = other_roc.execrecord - - self.assertRaisesRegexp(ValidationError, - re.escape('RunOutputCable "{}" has not decided whether or not to reuse an ExecRecord; ' - 'execrecord should not be set yet'.format(self.E31_42_ROC)), - self.E31_42_ROC.clean) - - def test_ROC_clean_reused_with_data(self): - """Reused is True but data is associated. - - A RunOutputCable which is reusing an ExecRecord should not have - generated any Datasets. - """ - self.step_through_roc_creation("roc_started") - self.E31_42_ROC.reused = True - self.singlet_dataset.file_source = self.E31_42_ROC - self.singlet_dataset.save() - self.assertRaisesRegexp(ValidationError, - re.escape('RunOutputCable "{}" reused an ExecRecord and should not have generated any ' - 'Datasets'.format(self.E31_42_ROC)), - self.E31_42_ROC.clean) - - def test_ROC_clean_not_reused_trivial_no_data(self): - """Reused is False, cable is trivial, no data associated. - - A RunOutputCable which is not reusing an ExecRecord, but which - is trivial, should not have generated any Datasets. - """ - self.step_through_roc_creation("roc_started") - self.E31_42_ROC.reused = False - self.assertIsNone(self.E31_42_ROC.clean()) - - def test_ROC_clean_not_reused_trivial_with_data(self): - """Reused is False, cable is trivial, data associated. - - A RunOutputCable which is not reusing an ExecRecord, but which - is trivial, should not have generated any Datasets. - """ - self.step_through_roc_creation("roc_started") - self.E31_42_ROC.reused = False - - cable_log = ExecLog(record=self.E31_42_ROC, invoking_record=self.E31_42_ROC, - start_time=timezone.now(), end_time=timezone.now()) - cable_log.save() - - self.singlet_dataset.file_source = self.E31_42_ROC - self.singlet_dataset.save() - self.assertRaisesRegexp(ValidationError, - re.escape('RunOutputCable "{}" is trivial and should not have generated any Datasets' - .format(self.E31_42_ROC)), - self.E31_42_ROC.clean) - - def test_ROC_clean_not_reused_nontrivial_with_data(self): - """Non-trivial cable, good data attached. - - A RunOutputCable which is non-trivial, is not reusing an - ExecRecord, and has data attached, is clean. - """ - self.step_through_roc_creation("custom_roc_completed") - self.assertIsNone(self.E21_41_ROC.clean()) - - def test_ROC_clean_not_reused_nontrivial_multiple_datasets(self): - """Non-trivial cable, multiple datasets attached. - - A RunOutputCable which is non-trivial, is not reusing an - ExecRecord, should generate at most one Dataset. - """ - self.step_through_roc_creation("custom_roc_completed") - self.E1_out_dataset.file_source = self.E21_41_ROC - self.E1_out_dataset.save() - self.assertRaisesRegexp(ValidationError, - re.escape('RunOutputCable "{}" should generate at most one Dataset' - .format(self.E21_41_ROC)), - self.E21_41_ROC.clean) - - def test_ROC_clean_not_reused_nontrivial_bad_data(self): - """Propagation: bad data attached to RunOutputCable. - - A RunOutputCable which produced a bad Dataset is not clean. - """ - self.step_through_roc_creation("custom_roc_completed") - old_checksum = self.doublet_dataset.MD5_checksum - self.doublet_dataset.MD5_checksum = "foo" - self.doublet_dataset.save() - - self.assertFalse(self.E21_41_ROC.reused) - self.assertFalse(self.E21_41_ROC.component.is_trivial()) - self.assertRaisesRegexp(ValidationError, - re.escape('File integrity of "{}" lost. Current checksum "{}" does not equal expected ' - 'checksum "{}"'.format(self.doublet_dataset, old_checksum, "foo")), - self.E21_41_ROC.clean) - - def test_ROC_clean_not_reused_incomplete_execrecord(self): - """Propagation: ExecRecord is not complete and clean. - - A trivial RunOutputCable which has an ExecRecord without the - appropriate ExecRecordIns is not clean. - """ - self.step_through_roc_creation("trivial_roc_completed") - self.E31_42_ROC.execrecord.execrecordins.first().delete() - self.assertRaisesRegexp(ValidationError, - re.escape('Input to ExecRecord "{}" is not quenched' - .format(self.E31_42_ROC.execrecord)), - self.E31_42_ROC.clean) - - def test_ROC_clean_incompatible_execrecord(self): - """RunOutputCable has ExecRecord for wrong PipelineOuputCable. - - The ExecRecord of a RunOutputCable must correspond to the same - PipelineOutputCable that the RunOutputCable corresponds to. - """ - self.step_through_roc_creation("custom_roc_completed") - self.E31_42_ROC.execrecord = self.E21_41_ROC.execrecord - err_msg = 'PipelineOutputCable of RunOutputCable "{}" is incompatible with the cable of its ExecRecord' - self.assertRaisesRegexp(ValidationError, re.escape(err_msg.format(self.E31_42_ROC)), self.E31_42_ROC.clean) - - def test_ROC_clean_wrong_object_execrecord(self): - """RunOutputCable has ExecRecord for PipelineStep. - - A RunOutputCable's ExecRecord must be for a PipelineOutputCable. - """ - self.step_through_roc_creation("custom_roc_completed") - runstep = self.step_E1.pipelinestep_instances.create(run=self.pE_run) - self.complete_RSICs(runstep, [self.raw_dataset], [self.raw_dataset]) - self.make_complete_non_reused(runstep, [self.raw_dataset], [self.doublet_dataset]) - self.E31_42_ROC.execrecord = runstep.execrecord - self.assertRaisesRegexp(ValidationError, - re.escape('ExecRecord of RunOutputCable "{}" does not represent a PipelineCable' - .format(self.E31_42_ROC)), - self.E31_42_ROC.clean) - - def test_ROC_clean_deleted_output_no_data(self): - """RunOutputCable with output marked for deletion has no data. - - A RunOutputCable from a subrun, where the PipelineStep has - marked the relevant output for deletion, should not have - any associated Datasets.""" - self.step_through_roc_creation("subrun") - self.step_E2.outputs_to_delete.add(self.pD.outputs.get(dataset_name="D1_out")) - self.assertIsNone(self.D11_21_ROC.clean()) - - def test_ROC_clean_deleted_output_with_data(self): - """RunOutputCable with output marked for deletion has data. - - A RunOutputCable from a subrun, where the PipelineStep has - marked the relevant output for deletion, should not have - any associated Datasets.""" - - self.step_through_roc_creation("subrun_complete") - self.step_E2.outputs_to_delete.add(self.pD.outputs.get(dataset_name="D1_out")) - - self.assertFalse(self.D11_21_ROC.keeps_output()) - self.assertTrue(self.D11_21_ROC.outputs.exists()) - self.assertRaisesRegexp(ValidationError, - re.escape('{} "{}" does not keep its output but a dataset was registered' - .format("RunOutputCable", self.D11_21_ROC)), - self.D11_21_ROC.clean) - - def test_ROC_clean_kept_output_no_data(self): - """RunOutputCable which should keep its output has no data. - - A RunOutputCable from a subrun, where the PipelineStep has - not marked the relevant output for deletion, should have a - Dataset in its ExecRecordOut. - """ - self.step_through_roc_creation("subrun_complete") - - # May 12, 2014: this caused the test to fail. We just want the ERO to not have - # data. - # self.triplet_3_rows_dataset.file_source = self.D11_21_ROC - # self.triplet_3_rows_dataset.save() - - self.C1_in_dataset.dataset_file.delete() - ero = self.D11_21_ROC.execrecord.execrecordouts.first() - self.assertRaisesRegexp(ValidationError, - re.escape('ExecRecordOut "{}" should reference existent data'.format(ero)), - self.D11_21_ROC.clean) - - def test_ROC_clean_kept_output_reused_no_data(self): - """Reused RunOutputCable should produce no data. - - A RunOutputCable from a subrun, where the PipelineStep has not - marked the relevant output for deletion, should have no data if - it is reusing an ExecRecord. - """ - self.step_through_roc_creation("subrun") - self.D11_21_ROC.reused = True - self.assertFalse(self.D11_21_ROC.outputs.exists()) - self.assertIsNone(self.D11_21_ROC.clean()) - - def test_ROC_clean_kept_output_trivial_no_data(self): - """Non-reused, trivial RunOutputCable should produce no data. - - A RunOutputCable from a subrun where the PipelineStep has not - marked the relevant output for deletion, and which is not - reusing an ExecRecord, should still have no data associated if - it is a trivial cable. - """ - self.step_through_roc_creation("subrun") - self.D11_21.custom_wires.all().delete() - self.assertFalse(self.D11_21_ROC.outputs.exists()) - self.assertIsNone(self.D11_21_ROC.clean()) - - def test_ROC_clean_kept_output_nontrivial_no_data(self): - """Non-reused, nontrivial RunOutputCable with no data. - - A nontrivial RunOutputCable from a subrun which is not reusing - an ExecRecord, where the PipelineStep has not marked the output - for deletion, should produce data. - """ - self.step_through_roc_creation("subrun") - self.make_complete_non_reused(self.D11_21_ROC, [self.C1_in_dataset], [self.C1_in_dataset]) - - self.assertTrue(self.D11_21_ROC.keeps_output()) - self.assertListEqual(self.D11_21_ROC.log.missing_outputs(), []) - self.assertFalse(self.D11_21_ROC.outputs.exists()) - self.assertRaisesRegexp(ValidationError, - re.escape('RunOutputCable "{}" was not reused, trivial, or deleted; it should have ' - 'produced data'.format(self.D11_21_ROC)), - self.D11_21_ROC.clean) - - def test_ROC_clean_wrong_data(self): - """Non-reused, nontrival RunOutputCable with the wrong Dataset. - - A RunOutputCable with a Dataset different from that in its - ExecRecordOut is not clean. - """ - self.step_through_roc_creation("subrun") - self.make_complete_non_reused(self.D11_21_ROC, [self.C1_in_dataset], [self.C1_in_dataset]) - self.triplet_3_rows_dataset.file_source = self.D11_21_ROC - self.triplet_3_rows_dataset.save() - - self.assertFalse(self.D11_21_ROC.component.is_trivial()) - self.assertFalse(self.D11_21_ROC.reused) - self.assertNotEqual(self.triplet_3_rows_dataset, - self.D11_21_ROC.execrecord.execrecordouts.first().dataset) - - self.assertRaisesRegexp(ValidationError, - re.escape('Dataset "{}" was produced by RunOutputCable "{}" but is not in an ERO of ' - 'ExecRecord "{}"'.format(self.triplet_3_rows_dataset, self.D11_21_ROC, - self.D11_21_ROC.execrecord)), - self.D11_21_ROC.clean) - - def test_ROC_clean_correct_data(self): - """Non-reused, nontrivial RunOutputCable with correct Dataset. - - A RunOutputCable with the same Dataset in its ExecRecordOut as - in its output, is clean. - """ - self.step_through_roc_creation("subrun_complete") - self.D11_21_ROC.outputs.add(self.C1_in_dataset) - self.assertIsNone(self.D11_21_ROC.clean()) - - def test_ROC_clean_trivial_with_data(self): - """Trivial top-level cable with associated data. - - A trivial RunOutputCable not for a subrun, which has an output - Dataset associated, is not clean. - """ - self.step_through_roc_creation("trivial_roc_completed") - self.singlet_dataset.file_source = self.E31_42_ROC - self.singlet_dataset.save() - self.assertRaisesRegexp(ValidationError, - re.escape('RunOutputCable "{}" is trivial and should not have generated any Datasets' - .format(self.E31_42_ROC)), - self.E31_42_ROC.clean) - - def test_ROC_clean_trivial_no_data(self): - """Trivial top-level cable with no associated data. - - A trivial RunOutputCable not for a subrun, which has no output - Dataset associated, is clean. - """ - self.step_through_roc_creation("trivial_roc_completed") - self.assertFalse(self.E31_42_ROC.has_data()) - self.assertIsNone(self.E31_42_ROC.clean()) - - def test_ROC_clean_nontrivial_good_data(self): - """Nontrivial top-level cable with correct associated data. - - A RunOutputCable with custom wires, which has associated output - data matching its ExecRecordOut, is clean. - """ - self.step_through_roc_creation("custom_roc_completed") - self.E21_41_ROC.outputs.add(self.doublet_dataset) - self.assertTrue(self.E21_41_ROC.has_data()) - self.assertIsNone(self.E21_41_ROC.clean()) - - def test_ROC_clean_nontrivial_no_data(self): - """Nontrivial top-level cable with no data associated. - - A nontrivial, non-reused RunOutputCable not for a subrun must - have produced output data, otherwise it is not clean. - """ - self.step_through_roc_creation("custom_roc_completed") - self.doublet_dataset.file_source = None - self.doublet_dataset.save() - self.assertRaisesRegexp(ValidationError, - re.escape('RunOutputCable "{}" was not reused, trivial, or deleted; it should have ' - 'produced data'.format(self.E21_41_ROC)), - self.E21_41_ROC.clean) - - def test_ROC_subrun_complete(self): - """Clean and complete RunOutputCable for subrun. - - A RunOutputCable for a subrun where the output of the - sub-pipeline kept, which has the correct associated Dataset, - is clean and complete. - """ - self.step_through_roc_creation("subrun_complete") - self.C1_in_dataset.file_source = self.D11_21_ROC - self.C1_in_dataset.save() - - self.assertIsNone(self.D11_21_ROC.clean()) - self.assertIsNotNone(self.D11_21_ROC.execrecord) - self.assertFalse(self.D11_21_ROC.reused) - - self.assertTrue(self.D11_21_ROC.is_complete()) - self.assertIsNone(self.D11_21_ROC.complete_clean()) - - def test_ROC_subrun_no_execrecord(self): - """RunOutputCable with no ExecRecord. - - A nontrivial RunOutputCable for a subrun which has no ExecRecord - is not complete yet. - """ - self.step_through_roc_creation("subrun") - self.D11_21_ROC.execrecord = None - self.assertFalse(self.D11_21_ROC.is_complete()) - self.assertRaisesRegexp(ValidationError, - re.escape('{} "{}" is not complete'.format("RunOutputCable", self.D11_21_ROC)), - self.D11_21_ROC.complete_clean) - - #### - # keeps_output tests added March 26, 2014 -- RL. - def test_ROC_keeps_output_top_level_trivial(self): - """ - A top-level trivial RunSIC should have keeps_output() return False. - """ - self.step_through_roc_creation("trivial_roc_completed") - self.assertFalse(self.E31_42_ROC.keeps_output()) - - def test_ROC_keeps_output_top_level_custom(self): - """ - A top-level custom RunSIC should have keeps_output() return True. - """ - self.step_through_roc_creation("custom_roc_completed") - self.assertTrue(self.E21_41_ROC.keeps_output()) - - def test_ROC_keeps_output_top_level_trivial_incomplete(self): - """ - A top-level trivial incomplete RunSIC should have keeps_output() return False. - """ - self.step_through_roc_creation("roc_started") - self.assertFalse(self.E31_42_ROC.keeps_output()) - - def test_ROC_keeps_output_top_level_custom_incomplete(self): - """ - A top-level custom incomplete RunSIC should have keeps_output() return True. - """ - self.step_through_roc_creation("roc_started") - self.assertTrue(self.E21_41_ROC.keeps_output()) - - def test_ROC_keeps_output_subrun_trivial_true(self): - """ - A trivial POC of a sub-run that doesn't discard its output should have keeps_output() return False. - """ - self.step_through_roc_creation("subrun_complete") - self.D11_21.custom_wires.all().delete() - self.assertFalse(self.D11_21_ROC.keeps_output()) - - def test_ROC_keeps_output_subrun_custom_true(self): - """ - A custom POC of a sub-run that doesn't discard its output should have keeps_output() return True. - """ - self.step_through_roc_creation("subrun_complete") - self.assertTrue(self.D11_21_ROC.keeps_output()) - - def test_ROC_keeps_output_subrun_custom_false(self): - """ - A custom POC of a sub-run that does discard its output should have keeps_output() return False. - """ - self.step_through_roc_creation("subrun_complete") - self.step_E2.add_deletion(self.D1_out) - self.assertFalse(self.D11_21_ROC.keeps_output()) - - def test_ROC_keeps_output_incomplete_subrun_trivial_true(self): - """ - A trivial POC of an incomplete sub-run that doesn't discard its output should have keeps_output() return False. - """ - self.step_through_roc_creation("subrun") - self.D11_21.custom_wires.all().delete() - self.assertFalse(self.D11_21_ROC.keeps_output()) - - def test_ROC_keeps_output_incomplete_subrun_custom_true(self): - """ - A custom cable of an incomplete sub-run that doesn't discard its output should have keeps_output() return True. - """ - self.step_through_roc_creation("subrun") - self.assertTrue(self.D11_21_ROC.keeps_output()) - - def test_ROC_keeps_output_incomplete_subrun_custom_false(self): - """ - A custom cable of an incomplete sub-run that does discard its output should have keeps_output() return False. - """ - self.step_through_roc_creation("subrun") - self.step_E2.add_deletion(self.D1_out) - self.assertFalse(self.D11_21_ROC.keeps_output()) - - -class ExecLogTests(ArchiveTestCase): - def test_delete_exec_log(self): - """Can delete an ExecLog.""" - step_E1_RS = self.step_E1.pipelinestep_instances.create(run=self.pE_run) - execlog = ExecLog(record=step_E1_RS, invoking_record=step_E1_RS) - execlog.save() - execlog.delete() - - def test_clean_record_not_RunComponent(self): - """record of ExecLog should be a RunComponent.""" - self.step_through_run_creation("outcables_done") - # Retrieve an ExecLog and point it at a subrun. - - for el in ExecLog.objects.all(): - original_record = el.record - el.record = self.step_E2_RS - self.assertRaisesRegexp(ValidationError, - 'ExecLog "{}" does not correspond to a Method or cable'.format(el), - el.clean) - el.record = original_record - el.save() - - def test_clean_record_and_invoked_records_different_Run(self): - """ExecLog's record and invoked_records should belong to the same top-level Run.""" - self.step_through_runstep_creation("first_rsic") - - other_run = self.pE.pipeline_instances.create(user=self.myUser) - other_run.grant_everyone_access() - self.make_complete_reused(self.step_E1_RS, [self.raw_dataset], [self.doublet_dataset], other_run) - - # Now step_E1_RS is marked as reused, and there is a dummy record of a RunStep belonging to - # other_run. Let's retrieve that ExecLog and mis-wire it so that its record belongs to self.pE_run - # while its invoking record remains other_run and that should trigger an error in ExecLog.clean(). - el_to_mess_up = other_run.runsteps.get(pipelinestep__step_num=1).log - - el_to_mess_up.record = self.step_E1_RS - - self.assertRaisesRegexp( - ValidationError, - 'ExecLog "{}" belongs to a different Run than its invoking RunStep/RSIC/ROC'.format(el_to_mess_up), - el_to_mess_up.clean) - - def test_clean_invoking_record_precedes_record_different_coords(self): - """An ExecLog should not have been invoked before its own record.""" - self.step_through_run_creation("outcables_done") - # Get all ExecLogs and change their invoking record to things preceding it in the run. - - for el in ExecLog.objects.all(): - # Skip if this ExecLog doesn't have top-level run pE_run. - if el.record.definite.top_level_run is not self.pE_run: - continue - - # Skip if this ExecLog has coordinates (1,). - if el.record.definite.get_coordinates() == (1,): - continue - - # Change invoking_record to step 1. - original_invoking_record = el.invoking_record - el.invoking_record = self.step_E1_RS - self.assertRaisesRegexp( - ValidationError, - 'ExecLog "{}" is invoked earlier than the RunStep/RSIC/ROC it belongs to'.format(el), - el.clean) - el.invoking_record = original_invoking_record - el.save() - - def test_clean_invoking_record_precedes_record_RSIC_of_RS(self): - """The ExecLog of a RunStep should not be invoked by its own RSICs.""" - self.step_through_run_creation("outcables_done") - - # Retrieve all ExecLogs of steps. - step_ELs = [] - for candidate_EL in ExecLog.objects.all(): - if candidate_EL.record.is_step(): - step_ELs.append(candidate_EL) - - for el in step_ELs: - original_invoking_record = el.invoking_record - for rsic in el.record.runstep.RSICs.all(): - el.invoking_record = rsic - self.assertRaisesRegexp( - ValidationError, - 'ExecLog "{}" is invoked earlier than the RunStep it belongs to'.format(el), - el.clean) - el.invoking_record = original_invoking_record - el.save() - - def test_clean_invoking_record_precedes_record_anything_precedes_ROC(self): - """The ExecLog of a RunOutputCable should not be invoked by anything before it in its run.""" - self.step_through_run_creation("outcables_done") - - # Retrieve all ExecLogs of ROCs. - ROC_ELs = [] - for candidate_EL in ExecLog.objects.all(): - if candidate_EL.record.is_outcable(): - ROC_ELs.append(candidate_EL) - - for el in ROC_ELs: - # Get its containing run. - containing_run = el.record.runoutputcable.run - - original_invoking_record = el.invoking_record - - # Change its invoking record to all steps and RSICs prior to it. - for step in containing_run.runsteps.all(): - - # Only use this step if it isn't a sub-pipeline. - if not step.has_subrun(): - el.invoking_record = step - self.assertRaisesRegexp( - ValidationError, - 'ExecLog "{}" is invoked earlier than the ROC it belongs to'.format(el), - el.clean) - - for rsic in step.RSICs.all(): - el.invoking_record = rsic - self.assertRaisesRegexp( - ValidationError, - 'ExecLog "{}" is invoked earlier than the ROC it belongs to'.format(el), - el.clean) - el.invoking_record = original_invoking_record - el.save() - - def test_clean_good_ExecLog(self): - """ - Test that clean doesn't barf on good ExecLogs. - """ - self.step_through_run_creation("outcables_done") - for el in ExecLog.objects.all(): - self.assertIsNone(el.clean()) - - def test_clean_good_ExecLog_invoked_later(self): - """ - ExecLogs can be invoked later in the same pipeline than their own record. - - Note that this case causes the containing Run and such to be inconsistent, - but we're only checking ExecLog right now. - """ - self.step_through_run_creation("outcables_done") - el_to_mess_with = self.step_E1_RS.log - - el_to_mess_with.invoking_record = self.step_E3_RS - - self.assertIsNone(el_to_mess_with.clean()) - - def test_is_successful_methodoutput_unset(self): - """ - An ExecLog with no MethodOutput should still be successful. - """ - self.step_through_runstep_creation("first_rsic") - execlog = ExecLog(record=self.step_E1_RS, invoking_record=self.step_E1_RS, - start_time=timezone.now(), end_time=None) - execlog.save() - self.assertTrue(execlog.is_successful()) - - def test_is_successful_methodoutput_return_code_unset(self): - """ - An ExecLog whose MethodOutput return_code has not been set yet should still be successful. - """ - self.step_through_runstep_creation("first_rsic") - execlog = ExecLog(record=self.step_E1_RS, invoking_record=self.step_E1_RS, - start_time=timezone.now(), end_time=None) - execlog.save() - mo = MethodOutput(execlog=execlog) - mo.save() - self.assertTrue(execlog.is_successful()) - - def test_is_successful_methodoutput_good(self): - """ - An ExecLog whose MethodOutput return_code is 0 should be successful. - """ - self.step_through_runstep_creation("first_rsic") - execlog = ExecLog(record=self.step_E1_RS, invoking_record=self.step_E1_RS, - start_time=timezone.now(), end_time=None) - execlog.save() - mo = MethodOutput(execlog=execlog, return_code=0) - mo.save() - self.assertTrue(execlog.is_successful()) - - def test_is_successful_methodoutput_bad(self): - """ - An ExecLog whose MethodOutput return_code is not 0 should be successful. - """ - self.step_through_runstep_creation("first_rsic") - execlog = ExecLog(record=self.step_E1_RS, invoking_record=self.step_E1_RS, - start_time=timezone.now(), end_time=None) - execlog.save() - mo = MethodOutput(execlog=execlog, return_code=1) - mo.save() - self.assertFalse(execlog.is_successful()) - - -@skipIfDBFeature('is_mocked') -class GetCoordinatesTests(TestCase, ArchiveTestCaseHelpers): - fixtures = ['archive_test_environment'] - """Tests of the get_coordinates functions of all Run and RunComponent classes.""" - - def test_get_coordinates_top_level_run(self): - """Coordinates of a top-level run should be an empty tuple.""" - self.step_through_run_creation("outcables_done") - top_level_runs = Run.objects.filter(parent_runstep=None) - for run in top_level_runs: - self.assertEquals(run.get_coordinates(), ()) - - def test_get_coordinates_subrun(self): - """Coordinates of a sub-run should match that of their parent runstep.""" - self.step_through_run_creation("outcables_done") - # pD_run is the second step of its containing top-level run. - self.assertEquals(self.pD_run.get_coordinates(), (2,)) - self.assertEquals(self.pD_run.get_coordinates(), self.step_E2_RS.get_coordinates()) - - def test_get_coordinates_top_level_step(self): - """Coordinates of a top-level step should be a one-entry tuple with its step number as the entry.""" - self.step_through_run_creation("outcables_done") - - top_level_steps = [] - for runstep in RunStep.objects.all(): - if runstep.run.parent_runstep is None: - top_level_steps.append(runstep) - - for top_level_step in top_level_steps: - self.assertEquals(top_level_step.get_coordinates(), - (top_level_step.pipelinestep.step_num,)) - - def test_get_coordinates_subrun_step(self): - """Coordinates of a subrun step should be a tuple lexicographically giving its location.""" - self.step_through_run_creation("outcables_done") - - # step_D1_RS (as defined by Eric) is at position (2,1). - self.assertEquals(self.step_D1_RS.get_coordinates(), (2, 1)) - - def test_get_coordinates_top_level_rsic(self): - """Coordinates of top-level RSICs should be one-entry tuples matching their parent RSs.""" - self.step_through_run_creation("outcables_done") - - for runstep in RunStep.objects.all(): - if runstep.run.parent_runstep is None: - # Examine the input cables. - for rsic in runstep.RSICs.all(): - self.assertEquals(rsic.get_coordinates(), (runstep.pipelinestep.step_num,)) - self.assertEquals(rsic.get_coordinates(), runstep.get_coordinates()) - - def test_get_coordinates_subrun_rsic(self): - """Coordinates of sub-run RSICs should match that of their parent runstep.""" - self.step_through_run_creation("outcables_done") - - # step_D1_RS (as defined by Eric) is at position (2,1). - for rsic in self.step_D1_RS.RSICs.all(): - self.assertEquals(rsic.get_coordinates(), (2, 1)) - self.assertEquals(rsic.get_coordinates(), self.step_D1_RS.get_coordinates()) - - def test_get_coordinates_top_level_roc(self): - """Coordinates of top-level ROCs should be empty tuples.""" - self.step_through_run_creation("outcables_done") - - for roc in RunOutputCable.objects.all(): - if roc.run.parent_runstep is None: - # Examine the cable. - self.assertEquals(roc.get_coordinates(), ()) - - def test_get_coordinates_subrun_roc(self): - """Coordinates of a subrun ROC should be the same as its parent run.""" - self.step_through_run_creation("outcables_done") - - # The second step is a sub-run. - for roc in self.pD_run.runoutputcables.all(): - self.assertEquals(roc.get_coordinates(), (2,)) - - -@skipIfDBFeature('is_mocked') -class GetCoordinatesOnDeepNestedRunTests(TestCase): - fixtures = ['deep_nested_run'] - - def test_get_coordinates_nested_runs(self): - """Test get_coordinates for a deeper-nested sub-run.""" - top_level_run = Run.objects.get(pipeline__family__name="p_top") - - self.assertEquals(top_level_run.get_coordinates(), ()) - - # Check all second-level and third-level runs. - for step in top_level_run.runsteps.all(): - first_lvl_step_num = step.pipelinestep.step_num - subrun = step.child_run - - self.assertEquals(subrun.get_coordinates(), (first_lvl_step_num,)) - - for substep in subrun.runsteps.all(): - second_lvl_step_num = substep.pipelinestep.step_num - basic_run = substep.child_run - self.assertEqual(basic_run.get_coordinates(), (first_lvl_step_num, second_lvl_step_num)) - - def test_get_coordinates_nested_runstep(self): - """Test get_coordinates for deeper-nested RunSteps.""" - top_level_run = Run.objects.get(pipeline__family__name="p_top") - - # Check all RunSteps of the top-level run and also their child and grandchild runs. - for step in top_level_run.runsteps.all(): - first_lvl_step_num = step.pipelinestep.step_num - self.assertEquals(step.get_coordinates(), (first_lvl_step_num,)) - - subrun = step.child_run - for substep in subrun.runsteps.all(): - second_lvl_step_num = substep.pipelinestep.step_num - self.assertEqual(substep.get_coordinates(), (first_lvl_step_num, second_lvl_step_num)) - - basic_run = substep.child_run - for basic_step in basic_run.runsteps.all(): - third_lvl_step_num = basic_step.pipelinestep.step_num - self.assertEqual(basic_step.get_coordinates(), - (first_lvl_step_num, second_lvl_step_num, third_lvl_step_num)) - - def test_get_coordinates_nested_rsic(self): - """Test get_coordinates for deeper-nested RSICs.""" - top_level_run = Run.objects.get(pipeline__family__name="p_top") - - # Check all RunSteps of the top-level run and also their child and grandchild runs. - for step in top_level_run.runsteps.all(): - first_lvl_step_num = step.pipelinestep.step_num - - for rsic in step.RSICs.all(): - self.assertEquals(rsic.get_coordinates(), (first_lvl_step_num,)) - - subrun = step.child_run - for substep in subrun.runsteps.all(): - second_lvl_step_num = substep.pipelinestep.step_num - - for subrsic in substep.RSICs.all(): - self.assertEqual(subrsic.get_coordinates(), (first_lvl_step_num, second_lvl_step_num)) - - basic_run = substep.child_run - for basic_step in basic_run.runsteps.all(): - third_lvl_step_num = basic_step.pipelinestep.step_num - - for basic_rsic in basic_step.RSICs.all(): - self.assertEqual(basic_rsic.get_coordinates(), - (first_lvl_step_num, second_lvl_step_num, third_lvl_step_num)) - - def test_get_coordinates_nested_roc(self): - """Test get_coordinates for deeper-nested sub-ROCs.""" - - top_level_run = Run.objects.get(pipeline__family__name="p_top") - - for roc in top_level_run.runoutputcables.all(): - self.assertEquals(roc.get_coordinates(), ()) - - # Check all second-level and third-level runs. - for step in top_level_run.runsteps.all(): - first_lvl_step_num = step.pipelinestep.step_num - subrun = step.child_run - - for subroc in subrun.runoutputcables.all(): - self.assertEquals(subroc.get_coordinates(), (first_lvl_step_num,)) - - for substep in subrun.runsteps.all(): - second_lvl_step_num = substep.pipelinestep.step_num - basic_run = substep.child_run - - for basic_roc in basic_run.runoutputcables.all(): - self.assertEqual(basic_roc.get_coordinates(), (first_lvl_step_num, second_lvl_step_num)) - - -class ExecLogIsCompleteIsSuccessfulTests(ArchiveTestCase): - """ - Tests the is_complete/is_successful functions of ExecLog. - - These functions are heavily dependent on each other, so we share the setups and test - both functions at the same time. - """ - def test_execlog_good_cases(self): - """ - Testing that all ExecLogs are complete and successful after a (simulated) good run. - """ - self.step_through_run_creation("outcables_done") - - for el in ExecLog.objects.all(): - if el.record.definite.top_level_run == self.pE_run: - self.assertTrue(el.is_complete()) - self.assertTrue(el.is_successful()) - - def test_execlog_has_not_ended_yet(self): - """ - Test on ExecLogs where has_ended() is False. - """ - self.step_through_run_creation("outcables_done") - - # Artificially change the logs' end_time to None. - for el in ExecLog.objects.all(): - if el.record.definite.top_level_run == self.pE_run: - orig_end_time = el.end_time - el.end_time = None - self.assertFalse(el.is_complete()) - self.assertTrue(el.is_successful()) - el.end_time = orig_end_time - - def test_execlog_cable_incomplete_successful(self): - """ - An incomplete cable's ExecLog should still be successful. - """ - self.step_through_run_creation("first_cable_created") - # No checks have been done yet, so the ExecLog is done but the RSIC is not. - step_E1_RSIC = self.step_E1_RS.RSICs.first() - self.make_complete_non_reused(step_E1_RSIC, [self.raw_dataset], [self.raw_dataset]) - - self.assertTrue(step_E1_RSIC.log.is_complete()) - self.assertTrue(step_E1_RSIC.log.is_successful()) - - def test_execlog_of_runstep_has_no_methodoutput(self): - """Test on ExecLogs for a RunStep that has no MethodOutput.""" - self.step_through_run_creation("first_cable") - - execlog = ExecLog(record=self.step_E1_RS, invoking_record=self.step_E1_RS, - start_time=timezone.now(), end_time=timezone.now()) - execlog.save() - # There is no MethodOutput defined. - - self.assertFalse(execlog.is_complete()) - self.assertTrue(execlog.is_successful()) - - def test_execlog_step_returncode_not_zero(self): - """Testing on an ExecLog of a RunStep whose Method has returned with code != 0.""" - self.step_through_run_creation("first_cable") - # Complete the RunStep... - self.make_complete_non_reused(self.step_E1_RS, [self.raw_dataset], [self.doublet_dataset]) - # ... and break it. - el_to_break = self.step_E1_RS.log - el_to_break.methodoutput.return_code = 1 - el_to_break.methodoutput.save() - self.assertTrue(self.step_E1_RS.log.is_complete()) - self.assertFalse(self.step_E1_RS.log.is_successful()) - - -@skipIfDBFeature('is_mocked') -class StateMachineActualExecutionTests(BaseTestCases.SlurmExecutionTestCase): - fixtures = ["archive_no_runs_test_environment"] - - def setUp(self): - install_fixture_files("archive_no_runs_test_environment") - tools.load_archive_no_runs_test_environment(self) - - def tearDown(self): - remove_fixture_files() - - def setup_incorrectly_random_method(self): - """ - Helper that sets up a CodeResource and Method that spits out the current time. - """ - python_code = """\ -#! /usr/bin/env python -import sys -import datetime -import csv - -with open(sys.argv[2], "w") as f: - dt_writer = csv.writer(f) - dt_writer.writerow(("year", "month", "day", "hour", "minute", "second", "microsecond")) - - curr_time = datetime.datetime.now() - dt_writer.writerow((curr_time.year, curr_time.month, curr_time.day, - curr_time.hour, curr_time.minute, curr_time.second, - curr_time.microsecond)) - -""" - self.curr_time_crr = tools.make_first_revision( - "CurrentTime", - "Gives the current time", - "CurrentTime.py", - python_code, - self.user_bob) - - self.curr_time_CDT = CompoundDatatype(user=self.user_bob) - self.curr_time_CDT.save() - self.curr_time_CDT.grant_everyone_access() - self.curr_time_CDT.members.create( - datatype=self.INT, - column_name="year", - column_idx=1 - ) - self.curr_time_CDT.members.create( - datatype=self.INT, - column_name="month", - column_idx=2 - ) - self.curr_time_CDT.members.create( - datatype=self.INT, - column_name="day", - column_idx=3 - ) - self.curr_time_CDT.members.create( - datatype=self.INT, - column_name="hour", - column_idx=4 - ) - self.curr_time_CDT.members.create( - datatype=self.INT, - column_name="minute", - column_idx=5 - ) - self.curr_time_CDT.members.create( - datatype=self.INT, - column_name="second", - column_idx=6 - ) - self.curr_time_CDT.members.create( - datatype=self.INT, - column_name="microsecond", - column_idx=7 - ) - - # Note that this is incorrectly marked as deterministic! (This is on purpose for the - # test.) - self.curr_time_method = tools.make_first_method( - "CurrentTime", - "Gives the current time -- incorrectly marked as deterministic", - self.curr_time_crr, - self.user_bob - ) - tools.simple_method_io(self.curr_time_method, self.curr_time_CDT, "ignored_input", "curr_time") - - self.time_noop = tools.make_first_method( - "TimeNoop", - "Noop on curr_time_CDT", - self.coderev_noop, - self.user_bob - ) - tools.simple_method_io(self.time_noop, self.curr_time_CDT, "input", "unchanged_output") - - self.time_trivial = tools.make_first_method( - "TimeTrivial", - "Also a noop on curr_time_CDT", - self.coderev_noop, - self.user_bob - ) - tools.simple_method_io(self.time_trivial, self.curr_time_CDT, "input", "unchanged_output") - - self.time_SD = tools.make_dataset( - """\ -year,month,day,hour,minute,second,microsecond -1969,1,1,0,0,0,0 -""", - self.curr_time_CDT, - True, - self.user_bob, - "EpochTime", - "12AM, January 1, 1969", - None, - True - ) - - @patch.object(Run, "quarantine", autospec=True, side_effect=Run.quarantine) - @patch.object(Run, "mark_failure", autospec=True, side_effect=Run.mark_failure) - @patch.object(Run, "stop", autospec=True, side_effect=Run.stop) - @patch.object(Run, "start", autospec=True, side_effect=Run.start) - @patch.object(RunComponent, "cancel_running", autospec=True, side_effect=RunComponent.cancel_running) - @patch.object(RunComponent, "cancel_pending", autospec=True, side_effect=RunComponent.cancel_pending) - @patch.object(RunComponent, "begin_recovery", autospec=True, side_effect=RunComponent.begin_recovery) - @patch.object(RunComponent, "quarantine", autospec=True, side_effect=RunComponent.quarantine) - @patch.object(RunComponent, "finish_failure", autospec=True, side_effect=RunComponent.finish_failure) - @patch.object(RunComponent, "finish_successfully", autospec=True, side_effect=RunComponent.finish_successfully) - @patch.object(RunComponent, "start", autospec=True, side_effect=RunComponent.start) - def test_runcomponent_unsuccessful_failed_invoked_log( - self, - mock_start, - mock_finish_successfully, - mock_finish_failure, - mock_quarantine, - mock_begin_recovery, - mock_cancel_pending, - mock_cancel_running, - mock_run_start, - mock_run_stop, - mock_run_mark_failure, - mock_run_quarantine - ): - """Testing of a RunComponent which has a failed invoked_log and never gets to its own execution.""" - - # Run two pipelines, the second of which reuses parts of the first, but the method has been - # screwed with in between. - p_one = tools.make_first_pipeline("p_one", "two no-ops", self.user_bob) - tools.create_linear_pipeline(p_one, [self.method_noop, self.method_noop], "p_one_in", "p_one_out") - p_one.create_outputs() - p_one.save() - # Mark the output of step 1 as not retained. - p_one.steps.get(step_num=1).add_deletion(self.method_noop.outputs.first()) - - # Set up a words dataset. - tools.make_words_dataset(self) - - run1 = Manager.execute_pipeline(self.user_bob, - p_one, - [self.dataset_words], - groups_allowed=[everyone_group()]).get_last_run() - - # All of the RunComponents should have been started. - run1_step1 = run1.runsteps.get(pipelinestep__step_num=1) - run1_step2 = run1.runsteps.get(pipelinestep__step_num=2) - run1_outcable = run1.runoutputcables.first() - mock_start.assert_has_calls([ - call(run1_step1), - call(run1_step1.RSICs.first()), - call(run1_step2), - call(run1_step2.RSICs.first()), - call(run1_outcable) - ]) - self.assertEquals(mock_start.call_count, 5) - - # All of them should have been finished successfully without event. - mock_finish_successfully.assert_has_calls([ - call(run1_step1.RSICs.first(), save=True), - call(run1_step1, save=True), - call(run1_step2.RSICs.first(), save=True), - call(run1_step2, save=True), - call(run1_outcable, save=True) - ]) - self.assertEquals(mock_finish_successfully.call_count, 5) - - # These were not called, so have not been mocked yet. - self.assertFalse(hasattr(mock_finish_failure, "assert_not_called")) - self.assertFalse(hasattr(mock_quarantine, "assert_not_called")) - self.assertFalse(hasattr(mock_begin_recovery, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_pending, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_running, "assert_not_called")) - - mock_run_start.assert_called_once_with(run1, save=True) - mock_run_stop.assert_called_once_with(run1, save=True) - self.assertFalse(hasattr(mock_run_mark_failure, "assert_not_called")) - - mock_run_start.reset_mock() - mock_run_stop.reset_mock() - mock_start.reset_mock() - mock_finish_successfully.reset_mock() - - # Oops! Between runs, self.method_noop gets screwed with. - with tempfile.TemporaryFile() as f: - f.write("#!/bin/bash\n exit 1".encode()) - os.remove(self.coderev_noop.content_file.path) - self.coderev_noop.content_file = File(f, name="bla") - self.coderev_noop.save() - - p_two = tools.make_first_pipeline("p_two", "one no-op then one trivial", self.user_bob) - tools.create_linear_pipeline(p_two, [self.method_noop, self.method_trivial], "p_two_in", "p_two_out") - p_two.create_outputs() - p_two.save() - # We also delete the output of step 1 so that it reuses the existing ER we'll have - # create for p_one. - p_two.steps.get(step_num=1).add_deletion(self.method_noop.outputs.first()) - - run2 = Manager.execute_pipeline(self.user_bob, p_two, [self.dataset_words], - groups_allowed=[everyone_group()]).get_last_run() - - # In the second run: the transformation of the second step should have tried to invoke the log of step 1 and - # failed. - run2_step1 = run2.runsteps.get(pipelinestep__step_num=1) - run2_step1_RSIC = run2_step1.RSICs.first() - run2_step2 = run2.runsteps.get(pipelinestep__step_num=2) - - # run2_step1 is failed, run2_step2 is cancelled. - self.assertTrue(run2_step1.is_failed()) - self.assertTrue(run2_step2.is_cancelled()) - self.assertTrue(run2.is_failed()) - - # Run 2 is failed, run 1 is quarantined, and run1_step1 is quarantined. - # run1_step2_RSIC, which was not affected by a failed data integrity check, - # should still be successful. - run1_step1 = run1.runsteps.get(pipelinestep__step_num=1) - self.assertTrue(run1_step1.is_quarantined()) - run1.refresh_from_db() - self.assertTrue(run1.is_quarantined()) - run1_step2_RSIC = run1.runsteps.get(pipelinestep__step_num=2).RSICs.first() - self.assertTrue(run1_step2_RSIC.is_successful()) - - self.assertFalse(run2_step2.has_log()) - self.assertEquals(run2_step2.invoked_logs.count(), 2) - self.assertEquals(set(run2_step2.invoked_logs.all()), {run2_step1.log, run2_step1_RSIC.log}) - - self.assertTrue(run2_step1_RSIC.log.is_successful()) - self.assertFalse(run2_step1.log.is_successful()) - - # The following RunComponents should have been started. - mock_start.assert_has_calls( - [ - call(run2_step1), - call(run2_step1.RSICs.first()), - call(run2_step2), - call(run2_step2.RSICs.first()) - ] - ) - self.assertEquals(mock_start.call_count, 4) - - # run2_step1 and its input cable attempted recovery. - mock_begin_recovery.assert_has_calls( - [ - call(run2_step1, save=True), - call(run2_step1.RSICs.first(), save=True) - ] - ) - self.assertEquals(mock_begin_recovery.call_count, 2) - - # The first step and cable finished successfully without event, and then the cable - # did again on recovery. - mock_finish_successfully.assert_has_calls( - [ - call(run2_step1.RSICs.first(), save=True), - call(run2_step1, save=True), - call(run2_step1.RSICs.first(), save=True) - ] - ) - self.assertEquals(mock_finish_successfully.call_count, 3) - - # run2_step1 failed on recovery. - mock_finish_failure.assert_called_once_with(run2_step1, save=True) - - # This stuff gets cancelled after the recovery fails. - mock_cancel_running.assert_has_calls( - [ - call(run2_step2.RSICs.first(), save=True), - call(run2_step2, save=True) - ] - ) - self.assertEquals(mock_cancel_running.call_count, 2) - - # run2 should have started, been marked as a failure, and then stopped. - mock_run_start.assert_called_once_with(run2, save=True) - mock_run_mark_failure.assert_called_once_with(run2, save=True) - mock_run_stop.assert_called_once_with(run2, save=True) - - # run1_step2 should have been quarantined. - mock_quarantine.assert_called_once_with(RunComponent.objects.get(pk=run1_step1.pk), - recurse_upward=True, save=True) - - # run1 should have been quarantined. - mock_run_quarantine.assert_called_once_with(run1, recurse_upward=True, save=True) - - def test_long_output(self): - """Should handle lots of output to stdout or stderr without deadlocking.""" - iteration_count = 100000 - python_code = """\ -#! /usr/bin/env python -import sys - -with open(sys.argv[2], "w") as f: - f.write("word\\n") - for i in range(%d): - print(i) - f.write("{}\\n".format(i)) -""" % iteration_count - expected_output = '\n'.join(map(str, range(iteration_count))) + '\n' - - code_revision = tools.make_first_revision( - "long_out", - "a script with lots of output", - "long_out.py", - python_code, - self.user_bob) - - # A Method telling Shipyard how to use the noop code on string data. - method = tools.make_first_method( - "string long_out", - "a method with lots of output", - code_revision, - self.user_bob) - tools.simple_method_io(method, self.cdt_string, "strings", "expected") - pipeline = tools.make_first_pipeline("pipe", "noisy", self.user_bob) - tools.create_linear_pipeline(pipeline, [method], "in", "out") - pipeline.create_outputs() - pipeline.save() - - # Set up a words dataset. - tools.make_words_dataset(self) - - active_run = Manager.execute_pipeline(self.user_bob, pipeline, [self.dataset_words], - groups_allowed=[everyone_group()]).get_last_run() - - run_step = active_run.runsteps.get(pipelinestep__step_num=1) - stdout_file = run_step.log.methodoutput.output_log - stdout_file.open(mode="r") - try: - stdout_content = stdout_file.read() - finally: - stdout_file.close() - - self.assertTrue(run_step.is_successful()) - self.assertTrue(run_step.log.is_successful()) - self.assertEqual(stdout_content, expected_output) - - @patch.object(Run, "quarantine", autospec=True, side_effect=Run.quarantine) - @patch.object(Run, "mark_failure", autospec=True, side_effect=Run.mark_failure) - @patch.object(Run, "stop", autospec=True, side_effect=Run.stop) - @patch.object(Run, "start", autospec=True, side_effect=Run.start) - @patch.object(RunComponent, "cancel_running", autospec=True, side_effect=RunComponent.cancel_running) - @patch.object(RunComponent, "cancel_pending", autospec=True, side_effect=RunComponent.cancel_pending) - @patch.object(RunComponent, "begin_recovery", autospec=True, side_effect=RunComponent.begin_recovery) - @patch.object(RunComponent, "quarantine", autospec=True, side_effect=RunComponent.quarantine) - @patch.object(RunComponent, "finish_failure", autospec=True, side_effect=RunComponent.finish_failure) - @patch.object(RunComponent, "finish_successfully", autospec=True, side_effect=RunComponent.finish_successfully) - @patch.object(RunComponent, "start", autospec=True, side_effect=RunComponent.start) - def test_runcomponent_unsuccessful_failed_integrity_check_during_recovery( - self, - mock_start, - mock_finish_successfully, - mock_finish_failure, - mock_quarantine, - mock_begin_recovery, - mock_cancel_pending, - mock_cancel_running, - mock_run_start, - mock_run_stop, - mock_run_mark_failure, - mock_run_quarantine - ): - """Testing of a RunComponent which has a failed integrity check during recovery.""" - - # Run two pipelines, the second of which reuses parts of the first, but the first step's output - # is different now. - self.setup_incorrectly_random_method() - - p_one = tools.make_first_pipeline("p_one", "time then noop", self.user_bob) - tools.create_linear_pipeline(p_one, [self.curr_time_method, self.time_noop], "p_one_in", "p_one_out") - p_one.create_outputs() - p_one.save() - # Mark the output of step 1 as not retained. - p_one.steps.get(step_num=1).add_deletion(self.curr_time_method.outputs.first()) - - run1 = Manager.execute_pipeline(self.user_bob, p_one, [self.time_SD]).get_last_run() - - # All of the RunComponents should have been started. - run1_step1 = run1.runsteps.get(pipelinestep__step_num=1) - run1_step2 = run1.runsteps.get(pipelinestep__step_num=2) - run1_outcable = run1.runoutputcables.first() - mock_start.assert_has_calls([ - call(run1_step1), - call(run1_step1.RSICs.first()), - call(run1_step2), - call(run1_step2.RSICs.first()), - call(run1_outcable) - ]) - self.assertEquals(mock_start.call_count, 5) - - # All of them should have been finished successfully without event. - mock_finish_successfully.assert_has_calls([ - call(run1_step1.RSICs.first(), save=True), - call(run1_step1, save=True), - call(run1_step2.RSICs.first(), save=True), - call(run1_step2, save=True), - call(run1_outcable, save=True) - ]) - self.assertEquals(mock_finish_successfully.call_count, 5) - - # These were not called, so have not been mocked yet. - self.assertFalse(hasattr(mock_finish_failure, "assert_not_called")) - self.assertFalse(hasattr(mock_quarantine, "assert_not_called")) - self.assertFalse(hasattr(mock_begin_recovery, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_pending, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_running, "assert_not_called")) - - mock_run_start.assert_called_once_with(run1, save=True) - mock_run_stop.assert_called_once_with(run1, save=True) - self.assertFalse(hasattr(mock_run_mark_failure, "assert_not_called")) - - mock_run_start.reset_mock() - mock_run_stop.reset_mock() - mock_start.reset_mock() - mock_finish_successfully.reset_mock() - - # Oops! The first step should not have been marked as deterministic. - p_two = tools.make_first_pipeline("p_two", "time then trivial", self.user_bob) - tools.create_linear_pipeline(p_two, [self.curr_time_method, self.time_trivial], "p_two_in", "p_two_out") - p_two.create_outputs() - p_two.save() - # We also delete the output of step 1 so that it reuses the existing ER we'll have - # created for p_one. - p_two.steps.get(step_num=1).add_deletion(self.curr_time_method.outputs.first()) - - run2 = Manager.execute_pipeline(self.user_bob, p_two, [self.time_SD]).get_last_run() - - # In the second run: the transformation of the second step should have tried to invoke the log of step 1 and - # failed. - run2_step1 = run2.runsteps.get(pipelinestep__step_num=1) - run2_step1_RSIC = run2_step1.RSICs.first() - run2_step2 = run2.runsteps.get(pipelinestep__step_num=2) - - self.assertTrue(run2_step1_RSIC.is_successful()) - self.assertTrue(run2_step1.is_failed()) - - # The corresponding step from run1 should also be quarantined, as should - # run1_step2_RSIC. run2_step2_RSIC is cancelled. - run1_step1 = run1.runsteps.get(pipelinestep__step_num=1) - self.assertTrue(run1_step1.is_quarantined()) - run1_step2_RSIC = run1.runsteps.get(pipelinestep__step_num=2).RSICs.first() - run2_step2_RSIC = run2.runsteps.get(pipelinestep__step_num=2).RSICs.first() - self.assertTrue(run1_step2_RSIC.is_quarantined()) - self.assertTrue(run2_step2_RSIC.is_cancelled()) - - # run2_step2, the recovering step, should be cancelled. - self.assertTrue(run2_step2.is_cancelled()) - - self.assertFalse(run2_step2.has_log()) - self.assertEquals(run2_step2.invoked_logs.count(), 2) - self.assertEquals(set(run2_step2.invoked_logs.all()), {run2_step1.log, run2_step1_RSIC.log}) - - self.assertTrue(run2_step1_RSIC.log.is_successful()) - self.assertTrue(run2_step1.log.is_successful()) - self.assertFalse(run2_step1.log.all_checks_passed()) - - # The following RunComponents should have been started. - mock_start.assert_has_calls( - [ - call(run2_step1), - call(run2_step1.RSICs.first()), - call(run2_step2), - call(run2_step2.RSICs.first()) - ] - ) - self.assertEquals(mock_start.call_count, 4) - - # run2_step1 and its input cable attempted recovery. - mock_begin_recovery.assert_has_calls( - [ - call(run2_step1, save=True), - call(run2_step1.RSICs.first(), save=True) - ] - ) - self.assertEquals(mock_begin_recovery.call_count, 2) - - # The first step and cable finished successfully without event, and then the cable - # did again on recovery. - mock_finish_successfully.assert_has_calls( - [ - call(run2_step1.RSICs.first(), save=True), - call(run2_step1, save=True), - call(run2_step1.RSICs.first(), save=True) - ] - ) - self.assertEquals(mock_finish_successfully.call_count, 3) - - # run2_step1 failed on recovery. - mock_finish_failure.assert_called_once_with(run2_step1, save=True) - - # This stuff gets cancelled after the recovery fails. - mock_cancel_running.assert_has_calls( - [ - call(run2_step2.RSICs.first(), save=True), - call(run2_step2, save=True) - ] - ) - self.assertEquals(mock_cancel_running.call_count, 2) - - # run2 should have started, been marked as a failure, and then stopped. - mock_run_start.assert_called_once_with(run2, save=True) - mock_run_mark_failure.assert_called_once_with(run2, save=True) - mock_run_stop.assert_called_once_with(run2, save=True) - - # run1_step1 should have been quarantined, along with run1_step2's input cable. - mock_quarantine.assert_has_calls( - [ - call(RunComponent.objects.get(pk=run1_step1.pk), recurse_upward=True, save=True), - call(RunComponent.objects.get(pk=run1_step2.RSICs.first().pk), recurse_upward=True, save=True) - ], - any_order=True - ) - - # run1 should have been quarantined. - mock_run_quarantine.assert_called_once_with(run1, recurse_upward=True, save=True) - - -@skipIfDBFeature('is_mocked') -class TopLevelRunTests(TestCase, ArchiveTestCaseHelpers): - fixtures = ['archive_test_environment'] - - def test_usual_run(self): - """Test on all elements of a simulated run.""" - self.step_through_run_creation("outcables_done") - - self.assertEquals(self.pE_run, self.pE_run.top_level_run) - - # All elements of both pE_run and pD_run should have top_level_run equal - # to pE_run. - - for curr_run in (self.pE_run, self.pD_run): - self.assertEquals(self.pE_run, curr_run.top_level_run) - - for runstep in curr_run.runsteps.all(): - self.assertEquals(self.pE_run, runstep.top_level_run) - - for cable_in in runstep.RSICs.all(): - self.assertEquals(self.pE_run, cable_in.top_level_run) - - for roc in curr_run.runoutputcables.all(): - self.assertEquals(self.pE_run, roc.top_level_run) - - -@skipIfDBFeature('is_mocked') -class TopLevelRunOnDeepNestedRunTests(TestCase): - fixtures = ['deep_nested_run'] - - def test_deep_nested_run(self): - """Test on all elements of a deep-nested run.""" - self.deep_nested_run = Run.objects.get( - pipeline__family__name='p_top') - - # Recurse down all elements of this run and make sure that they all have - # top_level_run equal to self.deep_nested_run. - top_level_runs = Run.objects.filter(pipeline__family__name="p_top") - second_level_runs = Run.objects.filter(pipeline__family__name="p_sub") - third_level_runs = Run.objects.filter(pipeline__family__name="p_basic") - - for curr_run in list(top_level_runs) + list(second_level_runs) + list(third_level_runs): - self.assertEquals(self.deep_nested_run, curr_run.top_level_run) - - for runstep in curr_run.runsteps.all(): - self.assertEquals(self.deep_nested_run, runstep.top_level_run) - - for cable_in in runstep.RSICs.all(): - self.assertEquals(self.deep_nested_run, cable_in.top_level_run) - - for roc in curr_run.runoutputcables.all(): - self.assertEquals(self.deep_nested_run, roc.top_level_run) - - -@skipIfDBFeature('is_mocked') -class RunStepDoNotReuseFailedExecRecordTests(BaseTestCases.SlurmExecutionTestCase): - def setUp(self): - tools.create_grandpa_sandbox_environment(self) - tools.make_words_dataset(self) - - def tearDown(self): - tools.destroy_grandpa_sandbox_environment(self) - - def test_failed_ER_is_not_reused(self): - """ - Failed ExecRecords are not reused. - """ - # The environment provides a method that always fails called method_fubar, which takes in data - # with CDT cdt_string (string: "word"), and puts out data with the same CDT in principle. - - failing_pipeline = tools.make_first_pipeline("failing pipeline", "a pipeline which always fails", - self.user_grandpa) - # self.method_fubar always exits with exit code 1, and creates no output. - tools.create_linear_pipeline( - failing_pipeline, - [self.method_fubar, self.method_noop], "indata", "outdata" - ) - failing_pipeline.create_outputs() - - first_step = failing_pipeline.steps.get(step_num=1) - first_step.add_deletion(self.method_fubar.outputs.first()) - - # This Pipeline is identical to the first but doesn't discard output. - failing_pl_2 = tools.make_first_pipeline("failing pipeline 2", "another pipeline which always fails", - self.user_grandpa) - tools.create_linear_pipeline( - failing_pl_2, - [self.method_fubar, self.method_noop], "indata", "outdata" - ) - failing_pl_2.create_outputs() - - # The first Pipeline should fail. The second will not reuse the first step's ExecRecord. - run_1 = Manager.execute_pipeline( - self.user_grandpa, - failing_pipeline, - [self.dataset_words], - groups_allowed=[everyone_group()] - ).get_last_run() - run_2 = Manager.execute_pipeline( - self.user_grandpa, - failing_pl_2, - [self.dataset_words], - groups_allowed=[everyone_group()] - ).get_last_run() - - failing_er = run_1.runsteps.get(pipelinestep__step_num=1).execrecord - self.assertNotEquals(failing_er, run_2.runsteps.get(pipelinestep__step_num=1).execrecord) - - self.assertEquals(failing_er.generator.methodoutput.return_code, 1) - self.assertFalse(failing_er.outputs_OK()) - - self.assertEquals(failing_er.execrecordouts.count(), 1) - produced_dataset = failing_er.execrecordouts.first().dataset - self.assertEquals(produced_dataset.content_checks.count(), 1) - self.assertFalse(produced_dataset.integrity_checks.exists()) - - bad_ccl = produced_dataset.content_checks.first() - self.assertEquals(bad_ccl, failing_er.generator.content_checks.first()) - - self.assertTrue(bad_ccl.baddata.missing_output) - - -@skipIfDBFeature('is_mocked') -class MethodOutputApiTests(BaseTestCases.ApiTestCase): - fixtures = ['simple_run'] - - def setUp(self): - super(MethodOutputApiTests, self).setUp() - - self.list_path = reverse("methodoutput-list") - self.detail_pk = 2 - self.detail_path = reverse("methodoutput-detail", - kwargs={'pk': self.detail_pk}) - self.output_redaction_path = reverse("methodoutput-output-redaction-plan", - kwargs={'pk': self.detail_pk}) - self.error_redaction_path = reverse("methodoutput-error-redaction-plan", - kwargs={'pk': self.detail_pk}) - self.code_redaction_path = reverse("methodoutput-code-redaction-plan", - kwargs={'pk': self.detail_pk}) - - self.list_view, _, _ = resolve(self.list_path) - self.detail_view, _, _ = resolve(self.detail_path) - self.output_redaction_view, _, _ = resolve(self.output_redaction_path) - self.error_redaction_view, _, _ = resolve(self.error_redaction_path) - self.code_redaction_view, _, _ = resolve(self.code_redaction_path) - - def test_list(self): - """ - Test the CompoundDatatype API list view. - """ - request = self.factory.get(self.list_path) - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - # There are four CDTs loaded into the Database by default. - self.assertEquals(len(response.data), 2) - self.assertEquals(response.data[0]['output_redacted'], False) - - def test_detail(self): - request = self.factory.get(self.detail_path) - force_authenticate(request, user=self.kive_user) - response = self.detail_view(request, pk=self.detail_pk) - self.assertEquals(response.data['error_redacted'], False) - - def test_output_redaction_plan(self): - request = self.factory.get(self.output_redaction_path) - force_authenticate(request, user=self.kive_user) - response = self.output_redaction_view(request, pk=self.detail_pk) - self.assertEquals(response.data['OutputLogs'], 1) - - def test_error_redaction_plan(self): - request = self.factory.get(self.error_redaction_path) - force_authenticate(request, user=self.kive_user) - response = self.error_redaction_view(request, pk=self.detail_pk) - self.assertEquals(response.data['ErrorLogs'], 1) - - def test_code_redaction_plan(self): - request = self.factory.get(self.code_redaction_path) - force_authenticate(request, user=self.kive_user) - response = self.code_redaction_view(request, pk=self.detail_pk) - self.assertEquals(response.data['ReturnCodes'], 1) - - def test_redaction(self): - request = self.factory.patch(self.detail_path, - {'output_redacted': "true"}) - force_authenticate(request, user=self.kive_user) - response = self.detail_view(request, pk=self.detail_pk) - self.assertEquals(response.status_code, status.HTTP_200_OK) - - method_output = MethodOutput.objects.get(pk=self.detail_pk) - self.assertTrue(method_output.is_output_redacted()) - - -@skipIfDBFeature('is_mocked') -class RunIncreasePermissionsNestedRunTests(TestCase): - """ - Tests of Run.increase_permissions_from_json with nested Runs. - """ - fixtures = ["deep_nested_run"] - - def setUp(self): - self.john = User.objects.get(username="john") - self.ringo = User.objects.get(username="ringo") - self.bob = User.objects.get(username="bob") - - # This Run has nesting two layers deep, and - # belongs to self.bob, with permissions granted to Everyone. - self.run = Run.objects.get(pipeline__family__name="p_top") - - # Let's sweep through and remove all extra permissions on anything self.bob ever produced. - for ds in Dataset.objects.filter(user=self.bob): - ds.groups_allowed.remove(everyone_group()) - - for run in Run.objects.filter(user=self.bob): - run.groups_allowed.remove(everyone_group()) - - def test_increase_permissions(self): - """ - Test granting permissions from a JSON input. - """ - perms_to_add = [ - [self.john.username, self.ringo.username], - [Group.objects.get(pk=groups.DEVELOPERS_PK).name] - ] - - self.run.increase_permissions_from_json(json.dumps(perms_to_add)) - - # Sweep through and make sure all outputs and Runs have had the appropriate - # permissions added. - for run in Run.objects.filter(user=self.bob): - if run.top_level_run == self.run: - self.assertTrue(run.users_allowed.filter(pk=self.john.pk).exists()) - self.assertTrue(run.users_allowed.filter(pk=self.ringo.pk).exists()) - self.assertTrue(run.groups_allowed.filter(pk=groups.DEVELOPERS_PK)) - - for ds in Dataset.objects.filter(user=self.bob, file_source__isnull=False): - if ds.file_source.top_level_run == self.run: - self.assertTrue(ds.users_allowed.filter(pk=self.john.pk).exists()) - self.assertTrue(ds.users_allowed.filter(pk=self.ringo.pk).exists()) - self.assertTrue(ds.groups_allowed.filter(pk=groups.DEVELOPERS_PK)) - - -@skipIfDBFeature('is_mocked') -class RunIncreasePermissionsCustomCableTests(TestCase): - """ - Tests of Run.increase_permissions_from_json with custom cables. - """ - fixtures = ["run_api_tests"] - - def setUp(self): - self.john = User.objects.get(username="john") - - # We want the inputs and the Pipeline to have appropriate permissions - # for us to be able to freely grant permissions on the Run. - for cdt in CompoundDatatype.objects.all(): - cdt.grant_everyone_access() - - for cr in CodeResource.objects.all(): - cr.grant_everyone_access() - for crr in cr.revisions.all(): - crr.grant_everyone_access() - - for mf in MethodFamily.objects.all(): - mf.grant_everyone_access() - for method in mf.members.all(): - method.grant_everyone_access() - - for pf in PipelineFamily.objects.all(): - pf.grant_everyone_access() - for pl in pf.members.all(): - pl.grant_everyone_access() - - self.pf = PipelineFamily.objects.get(name="self.pf") - self.pl = self.pf.members.get(revision_name="pX_revision_2") - - # This is the input to the run. - ds_structure = DatasetStructure.objects.get(dataset__name="pX_in_dataset", dataset__user=self.john) - input_ds = ds_structure.dataset - input_ds.grant_everyone_access() - - # This Run has nesting two layers deep, and - # belongs to self.bob, with permissions granted to Everyone. - self.run = Run.objects.get(pipeline__pk=self.pl.pk) - - def test_increase_permissions(self): - """ - Test granting permissions from a JSON input. - """ - perms_to_add = [[kive_user().username], [Group.objects.get(pk=groups.DEVELOPERS_PK).name]] - - self.run.increase_permissions_from_json(json.dumps(perms_to_add)) - - # Sweep through and make sure all outputs and Runs have had the appropriate - # permissions added. - for run in Run.objects.filter(user=self.john): - if run.top_level_run == self.run: - self.assertTrue(run.users_allowed.filter(pk=kive_user().pk).exists()) - self.assertTrue(run.groups_allowed.filter(pk=groups.DEVELOPERS_PK)) - - for ds in Dataset.objects.filter(user=self.john, file_source__isnull=False): - if ds.file_source.top_level_run == self.run: - self.assertTrue(ds.users_allowed.filter(pk=kive_user().pk).exists()) - self.assertTrue(ds.groups_allowed.filter(pk=groups.DEVELOPERS_PK)) - - -@skipIfDBFeature('is_mocked') -class GetAllAtomicRunComponentsTests(TestCase): - """ - Tests of Run.get_all_atomic_runcomponents. - """ - fixtures = ["deep_nested_run"] - - def setUp(self): - self.bob = User.objects.get(username="bob") - self.method_noop = Method.objects.get(family__name="string noop", revision_name="v1") - - # This Run has nesting two layers deep, and - # belongs to self.bob, with permissions granted to Everyone. - # It looks like: - # p_top - # - p_sub - # - p_basic - # - method_noop - # - method_noop - # - p_basic - # - p_sub - # - p_sub - # with single trivial cables connecting everything. - self.run = Run.objects.get(pipeline__family__name="p_top") - - # Let's sweep through and remove all extra permissions on anything self.bob ever produced. - for ds in Dataset.objects.filter(user=self.bob): - ds.groups_allowed.remove(everyone_group()) - - for run in Run.objects.filter(user=self.bob): - run.groups_allowed.remove(everyone_group()) - - def test_get_all_atomic_run_components(self): - """ - Test on a run with a fair amount of nesting. - """ - all_rcs = self.run.get_all_atomic_runcomponents() - - # The stuff that should be in all_rcs: - atomics = [] - # Look at each step of p_top. - for top_step_num in (1, 2, 3): - curr_top_step = self.run.runsteps.get(pipelinestep__step_num=top_step_num) - atomics += list(curr_top_step.RSICs.all()) - - # Descend into p_sub. - curr_p_sub_run = curr_top_step.child_run - for sub_step_num in (1, 2): - curr_sub_step = curr_p_sub_run.runsteps.get(pipelinestep__step_num=sub_step_num) - atomics += list(curr_sub_step.RSICs.all()) - - # Descend into p_basic. - curr_p_basic_run = curr_sub_step.child_run - for third_lvl_step_num in (1, 2): - curr_basic_step = curr_p_basic_run.runsteps.get(pipelinestep__step_num=third_lvl_step_num) - atomics += list(curr_basic_step.RSICs.all()) - atomics.append(curr_basic_step) - atomics += list(curr_p_basic_run.runoutputcables.all()) - - atomics += list(curr_p_sub_run.runoutputcables.all()) - - atomics += list(self.run.runoutputcables.all()) - - self.assertSetEqual(set(atomics), set(all_rcs)) - - -@skipIfDBFeature('is_mocked') -class EligiblePermissionsTests(TestCase): - fixtures = ["run_pipelines_recovering_reused_step"] - - def setUp(self): - self.john = User.objects.get(username="john") - self.ringo = User.objects.get(username="ringo") - self.bob = User.objects.get(username="bob") - - self.developers_group = Group.objects.get(pk=groups.DEVELOPERS_PK) - - self.p_one = Pipeline.objects.get(family__name="p_one", revision_name="v1") - self.p_two = Pipeline.objects.get(family__name="p_two", revision_name="v1") - - self.words_ds = Dataset.objects.get(name="blahblah") - - self.run_one = self.p_one.pipeline_instances.get(user=self.bob) - self.run_two = self.p_two.pipeline_instances.get(user=self.bob) - - self.run_two.groups_allowed.remove(everyone_group()) - self.run_one.groups_allowed.remove(everyone_group()) - self.words_ds.groups_allowed.remove(everyone_group()) - - self.p_two.groups_allowed.remove(everyone_group()) - self.p_one.groups_allowed.remove(everyone_group()) - - self.p_one.users_allowed.add(self.bob) - self.p_two.users_allowed.add(self.bob) - - def test_eligible_permissions_pipeline_inputs_restricted(self): - """ - Test retrieving eligible permissions on a Run whose Pipeline and inputs won't allow more permissions. - """ - addable_users, addable_groups = self.run_one.eligible_permissions() - - self.assertFalse(addable_users.exists()) - self.assertFalse(addable_groups.exists()) - - def test_eligible_permissions_pipeline_allows_more_but_inputs_restricted(self): - """ - Test retrieving eligible permissions when the Pipeline allows more but the input doesn't. - """ - self.p_one.groups_allowed.add(self.developers_group) - self.p_one.users_allowed.add(self.ringo) - - addable_users, addable_groups = self.run_one.eligible_permissions() - self.assertFalse(addable_users.exists()) - self.assertFalse(addable_groups.exists()) - - def test_eligible_permissions_pipeline_restricted_but_inputs_allow_more(self): - """ - Test retrieving eligible permissions when the input allows more but the Pipeline doesn't. - """ - self.words_ds.groups_allowed.add(self.developers_group) - self.words_ds.users_allowed.add(self.ringo) - - addable_users, addable_groups = self.run_one.eligible_permissions() - self.assertFalse(addable_users.exists()) - self.assertFalse(addable_groups.exists()) - - def test_eligible_permissions_pipeline_inputs_no_overlap(self): - """ - Test retrieving eligible permissions when the input and Pipeline have non-overlapping permissions. - """ - self.words_ds.groups_allowed.add(self.developers_group) - self.p_one.users_allowed.add(self.ringo) - - addable_users, addable_groups = self.run_one.eligible_permissions() - self.assertFalse(addable_users.exists()) - self.assertFalse(addable_groups.exists()) - - def test_eligible_permissions_with_eligible_users(self): - """ - Test retrieving eligible permissions when the input and Pipeline allow other users. - """ - self.words_ds.users_allowed.add(self.ringo) - self.p_one.users_allowed.add(self.ringo) - - addable_users, addable_groups = self.run_one.eligible_permissions() - self.assertSetEqual({self.ringo}, set(addable_users)) - self.assertFalse(addable_groups.exists()) - - def test_eligible_permissions_with_eligible_groups(self): - """ - Test retrieving eligible permissions when the input and Pipeline allow other groups. - """ - self.words_ds.groups_allowed.add(self.developers_group) - self.p_one.groups_allowed.add(everyone_group()) - - addable_users, addable_groups = self.run_one.eligible_permissions() - self.assertFalse(addable_users.exists()) - self.assertSetEqual({self.developers_group}, set(addable_groups)) - - def test_eligible_permissions_already_granted(self): - """ - Case where the input and Pipeline allow other users and groups that the Run already has. - """ - self.words_ds.users_allowed.add(self.ringo) - self.words_ds.groups_allowed.add(self.developers_group) - self.p_one.users_allowed.add(self.ringo) - self.p_one.groups_allowed.add(everyone_group()) - - self.run_one.users_allowed.add(self.ringo) - self.run_one.groups_allowed.add(self.developers_group) - - addable_users, addable_groups = self.run_one.eligible_permissions() - self.assertFalse(addable_users.exists()) - self.assertFalse(addable_groups.exists()) - - def test_eligible_permissions_reused_run_restricted(self): - """ - Case where the Run reuses steps from previous Runs and the original Run is restricted. - """ - self.words_ds.users_allowed.add(self.ringo) - self.words_ds.groups_allowed.add(self.developers_group) - self.p_one.users_allowed.add(self.ringo) - self.p_one.groups_allowed.add(everyone_group()) - self.p_two.users_allowed.add(self.ringo) - self.p_two.groups_allowed.add(everyone_group()) - - # This should still give no addable users or groups because self.run_one - # doesn't have added permissions. - addable_users, addable_groups = self.run_two.eligible_permissions() - self.assertFalse(addable_users.exists()) - self.assertFalse(addable_groups.exists()) - - def test_eligible_permissions_reused_run_permissions(self): - """ - Case where the Run reuses steps from previous Runs and the original Run had some permissions. - """ - self.words_ds.users_allowed.add(self.ringo, self.john) - self.words_ds.groups_allowed.add(self.developers_group) - self.p_one.users_allowed.add(self.ringo, self.john) - self.p_one.groups_allowed.add(everyone_group()) - self.p_two.users_allowed.add(self.ringo, self.john) - self.p_two.groups_allowed.add(everyone_group()) - - self.run_one.users_allowed.add(self.ringo) - self.run_one.groups_allowed.add(self.developers_group) - - # This should still give no addable users or groups because self.run_one - # doesn't have added permissions. - addable_users, addable_groups = self.run_two.eligible_permissions() - self.assertSetEqual({self.ringo}, set(addable_users)) - self.assertSetEqual({self.developers_group}, set(addable_groups)) - - def test_eligible_permissions_incomplete_run(self): - """ - Exception should be thrown when the run is incomplete. - """ - incomplete_run = Run( - user=self.bob, - name="IncompleteRun", - description="eligible_permissions should throw an exception", - pipeline=self.p_one - ) - self.assertRaisesRegexp( - RuntimeError, - "Eligible permissions cannot be found until the run is complete", - incomplete_run.eligible_permissions - ) - - -@skipIfDBFeature('is_mocked') -class CancelComponentsTests(TestCase): - """ - Tests of Run.cancel_unfinished and Run.cancel_unstarted. - - Indirectly also tests RunComponent.mark_cancelled. - """ - fixtures = ["deep_nested_run"] - - def setUp(self): - # This is stuff that's set up in the fixture. - self.user = User.objects.get(username='john') - # This is a nested pipeline, three layers deep. - self.p_top = Pipeline.objects.get(family__name="p_top") - self.p_sub = Pipeline.objects.get(family__name="p_sub") - self.p_basic = Pipeline.objects.get(family__name="p_basic") - self.words = Dataset.objects.get(name="blahblah") - - # Start an instance of the top-level Pipeline. - self.nested_run = self.p_top.pipeline_instances.create( - user=self.user, - name="FakeRun", - description="Dummy run used for testing cancel_unfinished and cancel_unstarted." - ) - self.nested_run.start(save=True) - self.nested_run.inputs.create(dataset=self.words, index=1) - - # Dummy up some RunComponents that are finished, in progress, and not started. - self.step_1 = self.nested_run.runsteps.create( - pipelinestep=self.p_top.steps.get(step_num=1) - ) - self.step_1.start() - self.step_1_ic = self.step_1.RSICs.create( - PSIC=self.step_1.pipelinestep.cables_in.first() - ) - self.step_1_ic.start() - self.step_1_ic.finish_successfully() - self.step_1_subrun = self.p_sub.pipeline_instances.create( - user=self.user, - parent_runstep=self.step_1 - ) - self.step_1_subrun.start() - self.step_11 = self.step_1_subrun.runsteps.create( - pipelinestep=self.p_sub.steps.get(step_num=1) - ) - self.step_11.start() - self.step_11_ic = self.step_11.RSICs.create( - PSIC=self.step_11.pipelinestep.cables_in.first() - ) - self.step_11_ic.start() - self.step_11_ic.finish_successfully() - self.step_11_subrun = self.p_basic.pipeline_instances.create( - user=self.user, - parent_runstep=self.step_11 - ) - self.step_11_subrun.start() - - self.step_111 = self.step_11_subrun.runsteps.create( - pipelinestep=self.p_basic.steps.get(step_num=1) - ) - self.step_111.start() - self.step_111_ic = self.step_111.RSICs.create( - PSIC=self.step_111.pipelinestep.cables_in.first() - ) - self.step_111_ic.start() - self.step_111_ic.finish_successfully() - self.step_111.finish_successfully() - self.step_112 = self.step_11_subrun.runsteps.create( - pipelinestep=self.p_basic.steps.get(step_num=2) - ) - self.step_112.start() - self.step_112_ic = self.step_112.RSICs.create( - PSIC=self.step_112.pipelinestep.cables_in.first() - ) - self.step_112_ic.start() - self.step_112_ic.finish_successfully() - self.step_112.finish_successfully() - self.outcable_11 = self.step_11_subrun.runoutputcables.create( - pipelineoutputcable=self.p_basic.outcables.first() - ) - self.outcable_11.start() - self.outcable_11.finish_successfully() - self.step_11_subrun.stop() - self.step_11.finish_successfully() - - self.step_12 = self.step_1_subrun.runsteps.create( - pipelinestep=self.p_sub.steps.get(step_num=2) - ) - self.step_12.start() - self.step_12_subrun = self.p_basic.pipeline_instances.create( - user=self.user, - parent_runstep=self.step_12 - ) - self.step_12_subrun.start() - self.step_121 = self.step_12_subrun.runsteps.create( - pipelinestep=self.p_basic.steps.get(step_num=1) - ) - self.step_121.start() - self.step_122 = self.step_12_subrun.runsteps.create( - pipelinestep=self.p_basic.steps.get(step_num=2) - ) - self.step_122.start() - self.outcable_12 = self.step_12_subrun.runoutputcables.create( - pipelineoutputcable=self.p_basic.outcables.first() - ) - self.outcable_12.start() - - # Step 2 has parts that are started and parts that are not. - self.step_2 = self.nested_run.runsteps.create( - pipelinestep=self.p_top.steps.get(step_num=2) - ) - self.step_2.start() - self.step_2_subrun = self.p_sub.pipeline_instances.create( - user=self.user, - parent_runstep=self.step_2 - ) - self.step_2_subrun.start() - self.step_21 = self.step_2_subrun.runsteps.create( - pipelinestep=self.p_sub.steps.get(step_num=1) - ) - self.step_21.start() - self.step_21_subrun = self.p_basic.pipeline_instances.create( - user=self.user, - parent_runstep=self.step_21 - ) - self.step_21_subrun.start() - - self.step_211 = self.step_21_subrun.runsteps.create( - pipelinestep=self.p_basic.steps.get(step_num=1) - ) - self.step_211.start() - self.step_212 = self.step_21_subrun.runsteps.create( - pipelinestep=self.p_basic.steps.get(step_num=2) - ) - self.step_212.start() - - self.step_22 = self.step_2_subrun.runsteps.create( - pipelinestep=self.p_sub.steps.get(step_num=2) - ) - self.step_22_subrun = self.p_basic.pipeline_instances.create( - user=self.user, - parent_runstep=self.step_22 - ) - self.step_221 = self.step_22_subrun.runsteps.create( - pipelinestep=self.p_basic.steps.get(step_num=1) - ) - self.step_222 = self.step_22_subrun.runsteps.create( - pipelinestep=self.p_basic.steps.get(step_num=2) - ) - - # Step 3 has not been started at all. - self.step_3 = self.nested_run.runsteps.create( - pipelinestep=self.p_top.steps.get(step_num=3) - ) - self.step_3_subrun = self.p_sub.pipeline_instances.create( - user=self.user, - parent_runstep=self.step_3 - ) - self.step_31 = self.step_3_subrun.runsteps.create( - pipelinestep=self.p_sub.steps.get(step_num=1) - ) - self.step_31_subrun = self.p_basic.pipeline_instances.create( - user=self.user, - parent_runstep=self.step_31 - ) - - self.step_311 = self.step_31_subrun.runsteps.create( - pipelinestep=self.p_basic.steps.get(step_num=1) - ) - self.step_312 = self.step_31_subrun.runsteps.create( - pipelinestep=self.p_basic.steps.get(step_num=2) - ) - - self.step_32 = self.step_3_subrun.runsteps.create( - pipelinestep=self.p_sub.steps.get(step_num=2) - ) - self.step_32_subrun = self.p_basic.pipeline_instances.create( - user=self.user, - parent_runstep=self.step_32 - ) - self.step_321 = self.step_32_subrun.runsteps.create( - pipelinestep=self.p_basic.steps.get(step_num=1) - ) - self.step_322 = self.step_32_subrun.runsteps.create( - pipelinestep=self.p_basic.steps.get(step_num=2) - ) - - self.successful = [ - self.step_1_ic, - self.step_11_ic, - self.step_111_ic, - self.step_111, - self.step_112_ic, - self.step_112, - self.outcable_11, - self.step_11 - ] - - self.successful_runs = [ - self.step_11_subrun - ] - - self.running = [ - self.step_1, - self.step_12, - self.step_121, - self.step_122, - self.outcable_12, - self.step_2, - self.step_21, - self.step_211, - self.step_212, - ] - - self.running_runs = [ - self.step_1_subrun, - self.step_12_subrun, - self.step_2_subrun, - self.step_21_subrun - ] - - self.pending = [ - self.step_22, - self.step_221, - self.step_222, - self.step_3, - self.step_31, - self.step_311, - self.step_312, - self.step_32, - self.step_321, - self.step_322, - ] - - self.pending_runs = [ - self.step_22_subrun, - self.step_3_subrun, - self.step_31_subrun, - self.step_32_subrun - ] - - def test_cancel_everything(self): - """ - Any unfinished RunComponents should be cancelled; others should be unaffected. - """ - self.nested_run.cancel_components() - - for rc in self.running: - rc.refresh_from_db() - self.assertTrue(rc.is_cancelled()) - - for run in self.running_runs: - run.refresh_from_db() - self.assertTrue(run.is_cancelled()) - - for rc in self.pending: - rc.refresh_from_db() - self.assertTrue(rc.is_cancelled()) - - for run in self.pending_runs: - run.refresh_from_db() - self.assertTrue(run.is_cancelled()) - - for rc in self.successful: - rc.refresh_from_db() - self.assertTrue(rc.is_successful()) - - for run in self.successful_runs: - run.refresh_from_db() - self.assertTrue(run.is_successful()) - - def test_cancel_except_step_122(self): - """ - Everything should be cancelled except for step_1, step_12, and step_122. - """ - self.nested_run.cancel_components(except_steps=[self.step_122]) - - exempted = [self.step_1, self.step_12, self.step_122] - exempted_subruns = [self.step_1_subrun, self.step_12_subrun] - - for rc in set(self.running) - set(exempted): - rc.refresh_from_db() - self.assertTrue(rc.is_cancelled()) - - for rc in exempted: - rc.refresh_from_db() - self.assertTrue(rc.is_running()) - - for run in set(self.running_runs) - set(exempted_subruns): - run.refresh_from_db() - self.assertTrue(run.is_cancelled()) - - for run in exempted_subruns: - run.refresh_from_db() - self.assertTrue(run.is_cancelling()) - - for rc in self.pending: - rc.refresh_from_db() - self.assertTrue(rc.is_cancelled()) - - for run in self.pending_runs: - run.refresh_from_db() - self.assertTrue(run.is_cancelled()) - - for rc in self.successful: - rc.refresh_from_db() - self.assertTrue(rc.is_successful()) - - for run in self.successful_runs: - run.refresh_from_db() - self.assertTrue(run.is_successful()) - - def test_cancel_except_outcable_12(self): - """ - Everything should be cancelled except for step_1, step_12, and outcable_12. - """ - self.nested_run.cancel_components(except_outcables=[self.outcable_12]) - - exempted = [self.step_1, self.step_12, self.outcable_12] - exempted_subruns = [self.step_1_subrun, self.step_12_subrun] - - for rc in set(self.running) - set(exempted): - rc.refresh_from_db() - self.assertTrue(rc.is_cancelled()) - - for rc in exempted: - rc.refresh_from_db() - self.assertTrue(rc.is_running()) - - for run in set(self.running_runs) - set(exempted_subruns): - run.refresh_from_db() - self.assertTrue(run.is_cancelled()) - - for run in exempted_subruns: - run.refresh_from_db() - self.assertTrue(run.is_cancelling()) - - for rc in self.pending: - rc.refresh_from_db() - self.assertTrue(rc.is_cancelled()) - - for run in self.pending_runs: - run.refresh_from_db() - self.assertTrue(run.is_cancelled()) - - for rc in self.successful: - rc.refresh_from_db() - self.assertTrue(rc.is_successful()) - - for run in self.successful_runs: - run.refresh_from_db() - self.assertTrue(run.is_successful()) - - def test_cancel_except_step_12_ic(self): - """ - Everything should be cancelled except for step_1 and the input cable to step_12. - """ - step_12_ic = self.step_12.RSICs.create( - PSIC=self.step_12.pipelinestep.cables_in.first() - ) - step_12_ic.start() - - self.nested_run.cancel_components(except_incables=[step_12_ic]) - - exempted = [self.step_1, step_12_ic] - exempted_subruns = [self.step_1_subrun] - - for rc in set(self.running) - set(exempted): - rc.refresh_from_db() - self.assertTrue(rc.is_cancelled()) - - for rc in exempted: - rc.refresh_from_db() - self.assertTrue(rc.is_running()) - - for run in set(self.running_runs) - set(exempted_subruns): - run.refresh_from_db() - self.assertTrue(run.is_cancelled()) - - for run in exempted_subruns: - run.refresh_from_db() - self.assertTrue(run.is_cancelling()) - - for rc in self.pending: - rc.refresh_from_db() - self.assertTrue(rc.is_cancelled()) - - for run in self.pending_runs: - run.refresh_from_db() - self.assertTrue(run.is_cancelled()) - - for rc in self.successful: - rc.refresh_from_db() - self.assertTrue(rc.is_successful()) - - for run in self.successful_runs: - run.refresh_from_db() - self.assertTrue(run.is_successful()) - - -@mocked_relations(RunBatch, RunState) -class RunBatchTests(TestCase): - def test_all_runs_complete_true(self): - """ - Testing when all runs are complete. - """ - rb = RunBatch() - - run1 = Run(_runstate_id=runstates.SUCCESSFUL_PK) - run2 = Run(_runstate_id=runstates.FAILED_PK) - run3 = Run(_runstate_id=runstates.QUARANTINED_PK) - - rb.runs.add(run1, run2, run3) - - self.assertTrue(rb.all_runs_complete()) - - def test_all_runs_complete_false(self): - """ - Testing when some runs are incomplete. - """ - rb = RunBatch() - - run1 = Run(_runstate_id=runstates.SUCCESSFUL_PK) - run2 = Run(_runstate_id=runstates.RUNNING_PK) - run3 = Run(_runstate_id=runstates.QUARANTINED_PK) - - rb.runs.add(run1, run2, run3) - - self.assertFalse(rb.all_runs_complete()) - - @mocked_relations(User, Group) - def test_eligible_permissions_no_runs(self): - """ - Testing that the eligible permissions on an empty RunBatch are everything. - """ - user_1 = User("Joe", "joe@ponzi.io", "joe", pk=1) - user_2 = User("Suzy", "suzy@ponzi.io", "suzy", pk=2) - expected_users = {user_1, user_2} - User.objects.add(*expected_users) - group_1 = Group("Floor 1", pk=101) - group_2 = Group("Floor 2", pk=102) - expected_groups = {group_1, group_2} - Group.objects.add(*expected_groups) - - rb = RunBatch() - eligible_users, eligible_groups = rb.eligible_permissions() - self.assertEqual(set(eligible_users), expected_users) - self.assertEqual(set(eligible_groups), expected_groups) - - def test_eligible_permissions_not_all_runs_complete(self): - """ - Testing eligible permissions raises an exception if not all runs are complete. - """ - rb = RunBatch() - rb.all_runs_complete = Mock(return_value=False) - self.assertRaisesRegexp( - RuntimeError, - "Eligible permissions cannot be found until all runs are complete", - rb.eligible_permissions - ) - - @mocked_relations(User, Group) - def test_eligible_permissions_runs_have_permissions(self): - """ - Testing the eligible permissions on a non-trivial RunBatch. - """ - user_1 = User("userone", "user1@ponzi.io", "user1", pk=1) - user_2 = User("usertwo", "user2@ponzi.io", "user2", pk=2) - user_3 = User("userthree", "user3@ponzi.io", "user3", pk=3) - User.objects.add(user_1, user_2, user_3) - - group_1 = Group(name="groupone", pk=101) - group_2 = Group(name="grouptwo", pk=102) - group_3 = Group(name="groupthree", pk=103) - Group.objects.add(group_1, group_2, group_3) - - rb = RunBatch() - - run1 = Run(_runstate_id=runstates.SUCCESSFUL_PK) - run2 = Run(_runstate_id=runstates.SUCCESSFUL_PK) - run3 = Run(_runstate_id=runstates.CANCELLED_PK) - - run1.eligible_permissions = Mock( - return_value=[ - MockSet(user_1, user_2), - MockSet(group_1, group_2) - ] - ) - run2.eligible_permissions = Mock( - return_value=[ - MockSet(user_2, user_3), - MockSet(group_2, group_3) - ] - ) - - # run3 has Everyone permissions, so should pose no limits. - run3.eligible_permissions = Mock( - return_value=[ - MockSet(user_1, user_2, user_3), - MockSet(group_1, group_2, group_3) - ] - ) - - rb.runs.add(run1, run2, run3) - - eligible_users, eligible_groups = rb.eligible_permissions() - self.assertEqual(set(eligible_users), {user_2}) - self.assertEqual(set(eligible_groups), {group_2}) - - # This run grants no one access. - run4 = Run(_runstate_id=runstates.SUCCESSFUL_PK) - run4.eligible_permissions = Mock( - return_value=[ - MockSet(), - MockSet() - ] - ) - rb.runs.add(run4) - - eligible_users, eligible_groups = rb.eligible_permissions() - self.assertFalse(eligible_users.exists()) - self.assertFalse(eligible_groups.exists()) diff --git a/kive/archive/tests_mock.py b/kive/archive/tests_mock.py deleted file mode 100644 index f8090389f..000000000 --- a/kive/archive/tests_mock.py +++ /dev/null @@ -1,1272 +0,0 @@ -from mock import Mock, patch - -from django.test import TestCase -from django.utils import timezone - -from django.contrib.auth.models import User -from django_mock_queries.query import MockSet -from django_mock_queries.mocks import mocked_relations - -from archive.models import Run, RunState, RunStep, RunOutputCable,\ - RunComponentState, ExecLog, RunInput, MethodOutput -from archive.serializers import RunOutputsSerializer -from constants import runstates, runcomponentstates -from librarian.models import ExecRecord, Dataset, ExecRecordOut -from pipeline.models import Pipeline, PipelineOutputCable, PipelineStep,\ - PipelineFamily -from transformation.models import TransformationInput, TransformationOutput, Transformation -from django.core.files.base import ContentFile - - -@mocked_relations(Run, RunState) -class RunStateMockTests(TestCase): - - def test_stop_running(self): - """ - Test that a Run properly transitions from Running to Successful when stopped. - """ - RunState.objects = MockSet(RunState(id=runstates.RUNNING_PK), RunState(id=runstates.SUCCESSFUL_PK)) - run = Run(_runstate_id=runstates.RUNNING_PK) - run.stop() - self.assertEqual(runstates.SUCCESSFUL_PK, run._runstate_id) - - def test_stop_running_quarantined_step(self): - """ - Test that a Run properly transitions from Running to Quarantined if a step is quarantined. - """ - RunState.objects = MockSet(RunState(id=runstates.RUNNING_PK), RunState(id=runstates.QUARANTINED_PK)) - run = Run(_runstate_id=runstates.RUNNING_PK) - - rs = RunStep(_runcomponentstate_id=runcomponentstates.QUARANTINED_PK) - run.runsteps.add(rs) - - run.stop() - self.assertEqual(runstates.QUARANTINED_PK, run._runstate_id) - - def test_stop_running_quarantined_outcable(self): - """ - Test that a Run properly transitions from Running to Quarantined if an outcable is quarantined. - """ - RunState.objects = MockSet(RunState(id=runstates.RUNNING_PK), RunState(id=runstates.QUARANTINED_PK)) - run = Run(_runstate_id=runstates.RUNNING_PK) - - roc = RunOutputCable(_runcomponentstate_id=runcomponentstates.QUARANTINED_PK) - run.runoutputcables.add(roc) - - run.stop() - self.assertEqual(runstates.QUARANTINED_PK, run._runstate_id) - - def test_stop_cancelling(self): - """ - Test that a Run properly transitions from Cancelling to Cancelled when stopped. - """ - RunState.objects = MockSet(RunState(id=runstates.CANCELLING_PK), RunState(id=runstates.CANCELLED_PK)) - run = Run(_runstate_id=runstates.CANCELLING_PK) - run.stop() - self.assertEqual(runstates.CANCELLED_PK, run._runstate_id) - - def test_stop_failing(self): - """ - Test that a Run properly transitions from Cancelling to Cancelled when stopped. - """ - RunState.objects = MockSet(RunState(id=runstates.FAILING_PK), RunState(id=runstates.FAILED_PK)) - run = Run(_runstate_id=runstates.FAILING_PK) - run.stop() - self.assertEqual(runstates.FAILED_PK, run._runstate_id) - - def test_cancel_pending(self): - """ - Test that a Run properly transitions from Pending to Cancelling on cancel(). - """ - RunState.objects = MockSet(RunState(id=runstates.PENDING_PK), RunState(id=runstates.CANCELLING_PK)) - run = Run(_runstate_id=runstates.PENDING_PK) - run.cancel() - self.assertEqual(runstates.CANCELLING_PK, run._runstate_id) - - def test_cancel_running(self): - """ - Test that a Run properly transitions from Running to Cancelling on cancel(). - """ - RunState.objects = MockSet(RunState(id=runstates.RUNNING_PK), RunState(id=runstates.CANCELLING_PK)) - run = Run(_runstate_id=runstates.RUNNING_PK) - run.cancel() - self.assertEqual(runstates.CANCELLING_PK, run._runstate_id) - - def test_mark_failure(self): - """ - Test that a Run properly transitions from Running to Failing on mark_failure(). - """ - RunState.objects = MockSet(RunState(id=runstates.RUNNING_PK), RunState(id=runstates.FAILING_PK)) - run = Run(_runstate_id=runstates.RUNNING_PK) - run.mark_failure() - self.assertEqual(runstates.FAILING_PK, run._runstate_id) - - def test_mark_failure_recurse_upward_no_parent_runstep(self): - """ - Test that a Run does not try to recurse upward when there's no parent_runstep. - """ - RunState.objects = MockSet(RunState(id=runstates.RUNNING_PK), RunState(id=runstates.FAILING_PK)) - run = Run(_runstate_id=runstates.RUNNING_PK) - # If this works, it didn't try to get at parent_runstep, which is None. - run.mark_failure(recurse_upward=True) - self.assertEqual(runstates.FAILING_PK, run._runstate_id) - - def test_mark_failure_recurse_upward(self): - """ - Test that mark_failure() can properly recurse upward. - """ - RunState.objects = MockSet(RunState(id=runstates.RUNNING_PK), RunState(id=runstates.FAILING_PK)) - run = Run(_runstate_id=runstates.RUNNING_PK) - - run_up_one_level = Run(_runstate_id=runstates.RUNNING_PK) - run_up_one_level.mark_failure = Mock() - - run.parent_runstep = RunStep(run=run_up_one_level) - - run.mark_failure(recurse_upward=True) - self.assertEqual(runstates.FAILING_PK, run._runstate_id) - run.parent_runstep.run.mark_failure.assert_called_once_with(save=True, recurse_upward=True) - - def test_mark_failure_no_recurse_upward(self): - """ - Test that mark_failure() does not recurse upward. - """ - RunState.objects = MockSet(RunState(id=runstates.RUNNING_PK), RunState(id=runstates.FAILING_PK)) - run = Run(_runstate_id=runstates.RUNNING_PK) - - run_up_one_level = Run(_runstate_id=runstates.RUNNING_PK) - run_up_one_level.mark_failure = Mock() - - run.parent_runstep = RunStep(run=run_up_one_level) - - run.mark_failure(recurse_upward=False) - self.assertEqual(runstates.FAILING_PK, run._runstate_id) - run.parent_runstep.run.mark_failure.assert_not_called() - - def test_begin_recovery(self): - """ - Test that a Run properly transitions from Successful to Running on begin_recovery(). - """ - RunState.objects = MockSet(RunState(id=runstates.RUNNING_PK), RunState(id=runstates.SUCCESSFUL_PK)) - run = Run(_runstate_id=runstates.SUCCESSFUL_PK, end_time=timezone.now()) - run.begin_recovery() - self.assertEqual(runstates.RUNNING_PK, run._runstate_id) - - def test_begin_recovery_recurse_upward_no_parent_runstep(self): - """ - Test that a Run properly transitions from Successful to Running on begin_recovery(). - """ - RunState.objects = MockSet(RunState(id=runstates.RUNNING_PK), RunState(id=runstates.SUCCESSFUL_PK)) - run = Run(_runstate_id=runstates.SUCCESSFUL_PK, end_time=timezone.now()) - # If this works, it didn't try to get at parent_runstep, which is None. - run.begin_recovery(recurse_upward=True) - self.assertEqual(runstates.RUNNING_PK, run._runstate_id) - - def test_begin_recovery_no_recurse_upward(self): - """ - Test that begin_recovery does not recurse upward. - """ - RunState.objects = MockSet(RunState(id=runstates.RUNNING_PK), RunState(id=runstates.SUCCESSFUL_PK)) - run = Run(_runstate_id=runstates.SUCCESSFUL_PK, end_time=timezone.now()) - - run_up_one_level = Run(_runstate_id=runstates.RUNNING_PK) - run_up_one_level.begin_recovery = Mock() - run.parent_runstep = RunStep(run=run_up_one_level) - - run.begin_recovery(recurse_upward=False) - self.assertEqual(runstates.RUNNING_PK, run._runstate_id) - run.parent_runstep.run.begin_recovery.assert_not_called() - - def test_begin_recovery_recurse_upward(self): - """ - Test that begin_recovery properly recurses upward. - """ - RunState.objects = MockSet(RunState(id=runstates.RUNNING_PK), RunState(id=runstates.SUCCESSFUL_PK)) - run = Run(_runstate_id=runstates.SUCCESSFUL_PK, end_time=timezone.now()) - - run_up_one_level = Run(_runstate_id=runstates.RUNNING_PK) - run_up_one_level.begin_recovery = Mock() - run.parent_runstep = RunStep(run=run_up_one_level) - - run.begin_recovery(recurse_upward=True) - self.assertEqual(runstates.RUNNING_PK, run._runstate_id) - run.parent_runstep.run.begin_recovery.assert_called_once_with(save=True, recurse_upward=True) - - def test_finish_recovery_successful(self): - """ - Test that a Run properly transitions from Running to Successful when recovery finishes. - """ - RunState.objects = MockSet(RunState(id=runstates.RUNNING_PK), RunState(id=runstates.SUCCESSFUL_PK)) - run = Run(_runstate_id=runstates.RUNNING_PK, end_time=timezone.now()) - run.is_running = Mock(return_value=True) - - run.finish_recovery() - self.assertEqual(runstates.SUCCESSFUL_PK, run._runstate_id) - - def test_finish_recovery_failed(self): - """ - Test that a Run properly transitions from Failing to Failed when recovery finishes. - """ - RunState.objects = MockSet(RunState(id=runstates.FAILING_PK), RunState(id=runstates.FAILED_PK)) - run = Run(_runstate_id=runstates.FAILING_PK, end_time=timezone.now()) - run.is_running = Mock(return_value=False) - run.is_failing = Mock(return_value=True) - - run.finish_recovery() - self.assertEqual(runstates.FAILED_PK, run._runstate_id) - - def test_finish_recovery_cancelled(self): - """ - Test that a Run properly transitions from Cancelling to Cancelled when recovery finishes. - """ - RunState.objects = MockSet(RunState(id=runstates.CANCELLING_PK), RunState(id=runstates.CANCELLED_PK)) - run = Run(_runstate_id=runstates.CANCELLING_PK, end_time=timezone.now()) - run.is_running = Mock(return_value=False) - run.is_failing = Mock(return_value=False) - - run.finish_recovery() - self.assertEqual(runstates.CANCELLED_PK, run._runstate_id) - - def test_finish_recovery_recurse_upward_no_parent_runstep(self): - """ - Test that a Run does not recurse upward when there's no parent_runstep. - """ - RunState.objects = MockSet(RunState(id=runstates.RUNNING_PK), RunState(id=runstates.SUCCESSFUL_PK)) - run = Run(_runstate_id=runstates.RUNNING_PK, end_time=timezone.now()) - run.is_running = Mock(return_value=True) - - # If this works, it didn't try to get at parent_runstep, which is None. - run.finish_recovery(recurse_upward=True) - self.assertEqual(runstates.SUCCESSFUL_PK, run._runstate_id) - - def test_finish_recovery_recurse_upward(self): - """ - Test that a Run properly recurses upward. - """ - RunState.objects = MockSet(RunState(id=runstates.RUNNING_PK), RunState(id=runstates.SUCCESSFUL_PK)) - run = Run(_runstate_id=runstates.RUNNING_PK, end_time=timezone.now()) - run.is_running = Mock(return_value=True) - - run_up_one_level = Run(_runstate_id=runstates.RUNNING_PK) - run_up_one_level.finish_recovery = Mock() - run.parent_runstep = RunStep(run=run_up_one_level) - - run.finish_recovery(recurse_upward=True) - self.assertEqual(runstates.SUCCESSFUL_PK, run._runstate_id) - run.parent_runstep.run.finish_recovery.assert_called_once_with(save=True, recurse_upward=True) - - def test_finish_recovery_no_recurse_upward(self): - """ - Test that a Run doesn't recurse upward. - """ - RunState.objects = MockSet(RunState(id=runstates.RUNNING_PK), RunState(id=runstates.SUCCESSFUL_PK)) - run = Run(_runstate_id=runstates.RUNNING_PK, end_time=timezone.now()) - run.is_running = Mock(return_value=True) - - run_up_one_level = Run(_runstate_id=runstates.RUNNING_PK) - run_up_one_level.finish_recovery = Mock() - run.parent_runstep = RunStep(run=run_up_one_level) - - run.finish_recovery(recurse_upward=False) - self.assertEqual(runstates.SUCCESSFUL_PK, run._runstate_id) - run.parent_runstep.run.finish_recovery.assert_not_called() - - def test_quarantine(self): - """ - Test quarantining of a Run. - """ - RunState.objects = MockSet(RunState(id=runstates.SUCCESSFUL_PK), RunState(id=runstates.QUARANTINED_PK)) - run = Run(_runstate_id=runstates.SUCCESSFUL_PK) - run.is_successful = Mock(return_value=True) - - run.quarantine(recurse_upward=False) - self.assertEqual(runstates.QUARANTINED_PK, run._runstate_id) - - def test_quarantine_recurse_upward_no_parent_runstep(self): - """ - Test quarantining of a Run does not recurse upward when no parent_runstep exists. - """ - RunState.objects = MockSet(RunState(id=runstates.SUCCESSFUL_PK), RunState(id=runstates.QUARANTINED_PK)) - run = Run(_runstate_id=runstates.SUCCESSFUL_PK) - run.is_successful = Mock(return_value=True) - - # This would fail if it tried to recurse upward. - run.quarantine(recurse_upward=True) - self.assertEqual(runstates.QUARANTINED_PK, run._runstate_id) - - def test_quarantine_no_recurse_upward(self): - """ - Test quarantining of a Run does not recurse upward when told not to. - """ - RunState.objects = MockSet(RunState(id=runstates.SUCCESSFUL_PK), RunState(id=runstates.QUARANTINED_PK)) - run = Run(_runstate_id=runstates.SUCCESSFUL_PK) - run.is_successful = Mock(return_value=True) - - run_up_one_level = Run(_runstate_id=runstates.SUCCESSFUL_PK) - run_up_one_level.is_successful = Mock(return_value=True) - run_up_one_level.quarantine = Mock() - run.parent_runstep = RunStep(run=run_up_one_level) - - run.quarantine(recurse_upward=False) - self.assertEqual(runstates.QUARANTINED_PK, run._runstate_id) - run.parent_runstep.run.is_successful.assert_not_called() - run.parent_runstep.run.quarantine.assert_not_called() - - def test_quarantine_recurse_upward_parent_run_not_successful(self): - """ - Test quarantining of a Run does not recurse upward when the parent run isn't successful. - """ - RunState.objects = MockSet(RunState(id=runstates.SUCCESSFUL_PK), RunState(id=runstates.QUARANTINED_PK)) - run = Run(_runstate_id=runstates.SUCCESSFUL_PK) - run.is_successful = Mock(return_value=True) - - run_up_one_level = Run(_runstate_id=runstates.SUCCESSFUL_PK) - run_up_one_level.is_successful = Mock(return_value=False) - run_up_one_level.quarantine = Mock() - run.parent_runstep = RunStep(run=run_up_one_level) - - run.quarantine(recurse_upward=True) - self.assertEqual(runstates.QUARANTINED_PK, run._runstate_id) - run.parent_runstep.run.is_successful.assert_called_once_with() - run.parent_runstep.run.quarantine.assert_not_called() - - def test_quarantine_recurse_upward(self): - """ - Test quarantining of a Run does not recurse upward when the parent run isn't successful. - """ - RunState.objects = MockSet(RunState(id=runstates.SUCCESSFUL_PK), RunState(id=runstates.QUARANTINED_PK)) - run = Run(_runstate_id=runstates.SUCCESSFUL_PK) - run.is_successful = Mock(return_value=True) - - run_up_one_level = Run(_runstate_id=runstates.SUCCESSFUL_PK) - run_up_one_level.is_successful = Mock(return_value=True) - run_up_one_level.quarantine = Mock() - run.parent_runstep = RunStep(run=run_up_one_level) - - run.quarantine(recurse_upward=True) - self.assertEqual(runstates.QUARANTINED_PK, run._runstate_id) - run.parent_runstep.run.is_successful.assert_called_once_with() - run.parent_runstep.run.quarantine.assert_called_once_with(save=True, recurse_upward=True) - - def test_attempt_decontamination(self): - """ - Test decontamination of a Run. - """ - RunState.objects = MockSet(RunState(id=runstates.SUCCESSFUL_PK), RunState(id=runstates.QUARANTINED_PK)) - run = Run(_runstate_id=runstates.QUARANTINED_PK) - run.is_quarantined = Mock(return_value=True) - - run.attempt_decontamination() - self.assertEqual(runstates.SUCCESSFUL_PK, run._runstate_id) - - def test_attempt_decontamination_does_nothing_when_runsteps_quarantined(self): - """ - Decontamination does nothing if a RunStep is still quarantined. - """ - RunState.objects = MockSet(RunState(id=runstates.SUCCESSFUL_PK), RunState(id=runstates.QUARANTINED_PK)) - run = Run(_runstate_id=runstates.QUARANTINED_PK) - run.is_quarantined = Mock(return_value=True) - - rs = RunStep(_runcomponentstate_id=runcomponentstates.QUARANTINED_PK) - run.runsteps.add(rs) - - run.attempt_decontamination() - self.assertEqual(runstates.QUARANTINED_PK, run._runstate_id) - - def test_attempt_decontamination_does_nothing_when_runoutputcables_quarantined(self): - """ - Decontamination does nothing if a RunOutputCable is still quarantined. - """ - RunState.objects = MockSet(RunState(id=runstates.SUCCESSFUL_PK), RunState(id=runstates.QUARANTINED_PK)) - run = Run(_runstate_id=runstates.QUARANTINED_PK) - run.is_quarantined = Mock(return_value=True) - - roc = RunOutputCable(run=run, _runcomponentstate_id=runcomponentstates.QUARANTINED_PK) - run.runoutputcables.add(roc) - - run.attempt_decontamination() - self.assertEqual(runstates.QUARANTINED_PK, run._runstate_id) - - def test_attempt_decontamination_recurse_upward_no_parent_run(self): - """ - Decontamination does not recurse upward if there's no parent run. - """ - RunState.objects = MockSet(RunState(id=runstates.SUCCESSFUL_PK), RunState(id=runstates.QUARANTINED_PK)) - run = Run(_runstate_id=runstates.QUARANTINED_PK) - run.is_quarantined = Mock(return_value=True) - - # This would fail if it actually recursed upward. - run.attempt_decontamination(recurse_upward=True) - self.assertEqual(runstates.SUCCESSFUL_PK, run._runstate_id) - - def test_attempt_decontamination_recurse_upward_parent_run_not_quarantined(self): - """ - Decontamination does not recurse upward if the parent run isn't quarantined. - """ - RunState.objects = MockSet(RunState(id=runstates.SUCCESSFUL_PK), RunState(id=runstates.QUARANTINED_PK)) - run = Run(_runstate_id=runstates.QUARANTINED_PK) - run.is_quarantined = Mock(return_value=True) - - run_up_one_level = Run() - run_up_one_level.is_quarantined = Mock(return_value=False) - run_up_one_level.attempt_decontamination = Mock() - run.parent_runstep = RunStep(run=run_up_one_level) - - run.attempt_decontamination(recurse_upward=True) - self.assertEqual(runstates.SUCCESSFUL_PK, run._runstate_id) - run.parent_runstep.run.attempt_decontamination.assert_not_called() - - def test_attempt_decontamination_recurse_upward(self): - """ - Test that decontamination recurses upward properly. - """ - RunState.objects = MockSet(RunState(id=runstates.SUCCESSFUL_PK), RunState(id=runstates.QUARANTINED_PK)) - run = Run(_runstate_id=runstates.QUARANTINED_PK) - run.is_quarantined = Mock(return_value=True) - - run_up_one_level = Run() - run_up_one_level.is_quarantined = Mock(return_value=True) - run_up_one_level.attempt_decontamination = Mock() - run.parent_runstep = RunStep(run=run_up_one_level) - - run.attempt_decontamination(recurse_upward=True) - self.assertEqual(runstates.SUCCESSFUL_PK, run._runstate_id) - run.parent_runstep.run.attempt_decontamination.assert_called_once_with(save=True, recurse_upward=True) - - -@mocked_relations(RunStep, RunComponentState) -class RunComponentStateMockTests(TestCase): - """ - Tests of RunComponent state transitions. - """ - - def test_start(self): - """ - Test start() of a RunComponent. - """ - RunComponentState.objects = MockSet( - RunComponentState(id=runcomponentstates.PENDING_PK), - RunComponentState(id=runcomponentstates.RUNNING_PK) - ) - rs = RunStep(_runcomponentstate_id=runcomponentstates.PENDING_PK) - rs.start() - self.assertEqual(rs._runcomponentstate_id, runcomponentstates.RUNNING_PK) - - def test_cancel_pending(self): - """ - Test cancel_pending() of a RunComponent. - """ - RunComponentState.objects = MockSet( - RunComponentState(id=runcomponentstates.PENDING_PK), - RunComponentState(id=runcomponentstates.CANCELLED_PK) - ) - rs = RunStep(_runcomponentstate_id=runcomponentstates.PENDING_PK) - rs.cancel_pending() - self.assertEqual(rs._runcomponentstate_id, runcomponentstates.CANCELLED_PK) - - def test_cancel_running(self): - """ - Test cancel_running() of a RunComponent. - """ - RunComponentState.objects = MockSet( - RunComponentState(id=runcomponentstates.RUNNING_PK), - RunComponentState(id=runcomponentstates.CANCELLED_PK) - ) - rs = RunStep(_runcomponentstate_id=runcomponentstates.RUNNING_PK) - rs.cancel_running() - self.assertEqual(rs._runcomponentstate_id, runcomponentstates.CANCELLED_PK) - - def test_cancel_when_pending(self): - """ - Test cancel() of a pending RunComponent. - """ - rs = RunStep(_runcomponentstate_id=runcomponentstates.PENDING_PK) - rs.cancel_pending = Mock() - rs.cancel_running = Mock() - rs.is_pending = Mock(return_value=True) - rs.cancel() - rs.cancel_pending.assert_called_once_with(save=True) - rs.cancel_running.assert_not_called() - - def test_cancel_when_running(self): - """ - Test cancel() of a running RunComponent. - """ - rs = RunStep(_runcomponentstate_id=runcomponentstates.RUNNING_PK) - rs.cancel_pending = Mock() - rs.cancel_running = Mock() - rs.is_pending = Mock(return_value=False) - rs.cancel() - rs.cancel_pending.assert_not_called() - rs.cancel_running.assert_called_once_with(save=True) - - def test_begin_recovery(self): - """ - Test begin_recovery(). - """ - RunComponentState.objects = MockSet( - RunComponentState(id=runcomponentstates.RUNNING_PK), - RunComponentState(id=runcomponentstates.SUCCESSFUL_PK) - ) - rs = RunStep(_runcomponentstate_id=runcomponentstates.SUCCESSFUL_PK, - run=Run()) - rs.has_ended = Mock(return_value=True) - rs.run.begin_recovery = Mock() - rs.begin_recovery() - self.assertEqual(rs._runcomponentstate_id, runcomponentstates.RUNNING_PK) - rs.run.begin_recovery.assert_not_called() - - def test_begin_recovery_recurse_upward(self): - """ - Test begin_recovery() recurses upward correctly. - """ - RunComponentState.objects = MockSet( - RunComponentState(id=runcomponentstates.RUNNING_PK), - RunComponentState(id=runcomponentstates.SUCCESSFUL_PK) - ) - rs = RunStep(_runcomponentstate_id=runcomponentstates.SUCCESSFUL_PK, - run=Run()) - rs.has_ended = Mock(return_value=True) - rs.run.begin_recovery = Mock() - rs.begin_recovery(recurse_upward=True) - self.assertEqual(rs._runcomponentstate_id, runcomponentstates.RUNNING_PK) - rs.run.begin_recovery.assert_called_once_with(save=True, recurse_upward=True) - - def test_finish_successfully(self): - """ - Test finish_successfully() on a RunComponent. - """ - RunComponentState.objects = MockSet( - RunComponentState(id=runcomponentstates.SUCCESSFUL_PK) - ) - rs = RunStep(_runcomponentstate_id=runcomponentstates.RUNNING_PK, - run=Run()) - rs.stop = Mock() - rs.has_ended = Mock(return_value=False) - - rs.finish_successfully() - self.assertEqual(rs._runcomponentstate_id, runcomponentstates.SUCCESSFUL_PK) - rs.stop.assert_called_once_with(save=False) - - def test_finish_successfully_recovery(self): - """ - Test finish_successfully() on a RunComponent that's recovering (i.e. has_ended() returns True). - """ - RunComponentState.objects = MockSet( - RunComponentState(id=runcomponentstates.SUCCESSFUL_PK) - ) - rs = RunStep(_runcomponentstate_id=runcomponentstates.RUNNING_PK, - run=Run()) - rs.stop = Mock() - rs.has_ended = Mock(return_value=True) - - rs.finish_successfully() - self.assertEqual(rs._runcomponentstate_id, runcomponentstates.SUCCESSFUL_PK) - rs.stop.assert_not_called() - - def test_finish_successfully_decontaminate(self): - """ - Test finish_successfully() on a RunComponent whose ExecRecord comes from a quarantined component. - """ - RunComponentState.objects = MockSet( - RunComponentState(id=runcomponentstates.SUCCESSFUL_PK) - ) - rs = RunStep(_runcomponentstate_id=runcomponentstates.RUNNING_PK, - run=Run()) - rs.stop = Mock() - rs.has_ended = Mock(return_value=False) - - generating_rs = RunStep() - generating_log = ExecLog(record=generating_rs) - rs.execrecord = ExecRecord(generator=generating_log) - generating_rs.is_quarantined = Mock(return_value=True) - rs.execrecord.decontaminate_runcomponents = Mock() - - rs.finish_successfully() - self.assertEqual(rs._runcomponentstate_id, runcomponentstates.SUCCESSFUL_PK) - rs.stop.assert_called_once_with(save=False) - - generating_rs.is_quarantined.assert_called_once_with() - rs.execrecord.decontaminate_runcomponents.assert_called_once_with() - - def test_finish_failure(self): - """ - Test finish_failure() on a RunComponent. - """ - RunComponentState.objects = MockSet( - RunComponentState(id=runcomponentstates.FAILED_PK) - ) - rs = RunStep(_runcomponentstate_id=runcomponentstates.RUNNING_PK, - run=Run()) - rs.stop = Mock() - rs.has_ended = Mock(return_value=False) - - rs.finish_failure() - self.assertEqual(rs._runcomponentstate_id, runcomponentstates.FAILED_PK) - rs.stop.assert_called_once_with(save=False) - - def test_finish_failure_recovery(self): - """ - Test finish_failure() on a RunComponent that's recovering (i.e. has already ended). - """ - RunComponentState.objects = MockSet( - RunComponentState(id=runcomponentstates.FAILED_PK) - ) - rs = RunStep(_runcomponentstate_id=runcomponentstates.RUNNING_PK, - run=Run()) - rs.stop = Mock() - rs.has_ended = Mock(return_value=True) - - rs.finish_failure() - self.assertEqual(rs._runcomponentstate_id, runcomponentstates.FAILED_PK) - rs.stop.assert_not_called() - - def test_finish_failure_recurse_upward_run_not_running(self): - """ - Test finish_failure() doesn't recurse upward if the parent run is not running. - """ - RunComponentState.objects = MockSet( - RunComponentState(id=runcomponentstates.FAILED_PK) - ) - rs = RunStep(_runcomponentstate_id=runcomponentstates.RUNNING_PK, - run=Run()) - rs.stop = Mock() - rs.has_ended = Mock(return_value=True) - rs.run.is_running = Mock(return_value=False) - rs.run.mark_failure = Mock() - - rs.finish_failure(recurse_upward=True) - self.assertEqual(rs._runcomponentstate_id, runcomponentstates.FAILED_PK) - rs.stop.assert_not_called() - rs.run.mark_failure.assert_not_called() - - def test_finish_failure_recurse_upward(self): - """ - Test finish_failure() properly recurses upward. - """ - RunComponentState.objects = MockSet( - RunComponentState(id=runcomponentstates.FAILED_PK) - ) - rs = RunStep(_runcomponentstate_id=runcomponentstates.RUNNING_PK, - run=Run()) - rs.stop = Mock() - rs.has_ended = Mock(return_value=False) - rs.run.is_running = Mock(return_value=True) - rs.run.mark_failure = Mock() - - rs.finish_failure(recurse_upward=True) - self.assertEqual(rs._runcomponentstate_id, runcomponentstates.FAILED_PK) - rs.stop.assert_called_once_with(save=False) - rs.run.mark_failure.assert_called_once_with(save=True, recurse_upward=True) - - def test_quarantine(self): - """ - Test quarantining a RunComponent. - """ - RunComponentState.objects = MockSet( - RunComponentState(id=runcomponentstates.QUARANTINED_PK) - ) - rs = RunStep(_runcomponentstate_id=runcomponentstates.SUCCESSFUL_PK, - run=Run()) - rs.stop = Mock() - rs.run.quarantine = Mock() - - rs.quarantine() - self.assertEqual(rs._runcomponentstate_id, runcomponentstates.QUARANTINED_PK) - rs.run.quarantine.assert_not_called() - - def test_quarantine_recurse_upward_run_not_successful(self): - """ - Test that no upward recursion occurs if the parent run isn't successful. - """ - RunComponentState.objects = MockSet( - RunComponentState(id=runcomponentstates.QUARANTINED_PK) - ) - rs = RunStep(_runcomponentstate_id=runcomponentstates.SUCCESSFUL_PK, - run=Run()) - rs.stop = Mock() - rs.run.quarantine = Mock() - rs.run.is_successful = Mock(return_value=False) - - rs.quarantine(recurse_upward=True) - self.assertEqual(rs._runcomponentstate_id, runcomponentstates.QUARANTINED_PK) - rs.run.is_successful.assert_called_once_with() - rs.run.quarantine.assert_not_called() - - def test_quarantine_recurse_upward(self): - """ - Test of upward recursion in quarantine. - """ - RunComponentState.objects = MockSet( - RunComponentState(id=runcomponentstates.QUARANTINED_PK) - ) - rs = RunStep(_runcomponentstate_id=runcomponentstates.SUCCESSFUL_PK, - run=Run()) - rs.stop = Mock() - rs.run.quarantine = Mock() - rs.run.is_successful = Mock(return_value=True) - - rs.quarantine(recurse_upward=True) - self.assertEqual(rs._runcomponentstate_id, runcomponentstates.QUARANTINED_PK) - rs.run.is_successful.assert_called_once_with() - rs.run.quarantine.assert_called_once_with(save=True, recurse_upward=True) - - def test_decontaminate(self): - """ - Test of decontamination of a RunComponent. - """ - RunComponentState.objects = MockSet( - RunComponentState(id=runcomponentstates.SUCCESSFUL_PK) - ) - rs = RunStep(_runcomponentstate_id=runcomponentstates.QUARANTINED_PK, - run=Run()) - rs.stop = Mock() - rs.run.attempt_decontamination = Mock() - rs.run.is_quarantined = Mock() - rs.run.refresh_from_db = Mock() - - rs.decontaminate() - self.assertEqual(rs._runcomponentstate_id, runcomponentstates.SUCCESSFUL_PK) - rs.run.is_quarantined.assert_not_called() - rs.run.attempt_decontamination.assert_not_called() - rs.run.refresh_from_db.assert_called_once_with() - - def test_decontaminate_recurse_upward_run_not_quarantined(self): - """ - Test decontamination does not recurse upward if parent run isn't quarantined. - """ - RunComponentState.objects = MockSet( - RunComponentState(id=runcomponentstates.SUCCESSFUL_PK) - ) - rs = RunStep(_runcomponentstate_id=runcomponentstates.QUARANTINED_PK, - run=Run()) - rs.stop = Mock() - rs.run.attempt_decontamination = Mock() - rs.run.is_quarantined = Mock(return_value=False) - rs.run.refresh_from_db = Mock() - - rs.decontaminate(recurse_upward=True) - self.assertEqual(rs._runcomponentstate_id, runcomponentstates.SUCCESSFUL_PK) - rs.run.is_quarantined.assert_called_once_with() - rs.run.attempt_decontamination.assert_not_called() - rs.run.refresh_from_db.assert_called_once_with() - - def test_decontaminate_recurse_upward(self): - """ - Test decontamination properly recurses upward. - """ - RunComponentState.objects = MockSet( - RunComponentState(id=runcomponentstates.SUCCESSFUL_PK) - ) - rs = RunStep(_runcomponentstate_id=runcomponentstates.QUARANTINED_PK, - run=Run()) - rs.stop = Mock() - rs.run.attempt_decontamination = Mock() - rs.run.is_quarantined = Mock(return_value=True) - rs.run.refresh_from_db = Mock() - - rs.decontaminate(recurse_upward=True) - self.assertEqual(rs._runcomponentstate_id, runcomponentstates.SUCCESSFUL_PK) - rs.run.is_quarantined.assert_called_once_with() - rs.run.attempt_decontamination.assert_called_once_with(save=True, recurse_upward=True) - rs.run.refresh_from_db.assert_called_once_with() - - -@mocked_relations(Run, Pipeline, Transformation, ExecRecord) -class RunOutputsSerializerMockTests(TestCase): - def test_no_steps(self): - run = Run(id=1234, pipeline=Pipeline()) - expected_data = {'output_summary': [], 'input_summary': [], 'id': 1234} - - data = RunOutputsSerializer(run).data - - self.assertEqual(expected_data, data) - - def test_inputs(self): - run = Run(id=1234, pipeline=Pipeline()) - run.inputs = MockSet(RunInput(index=1, dataset=Dataset(id=99)), - RunInput(index=2, dataset=Dataset(id=100))) - run.pipeline.inputs = MockSet(TransformationInput(dataset_idx=1, - dataset_name='foo'), - TransformationInput(dataset_idx=2)) - expected_data = {'output_summary': [], - 'input_summary': [{'is_ok': True, - 'errors': [], - 'name': 'foo', - 'url': '/api/datasets/99/', - 'is_invalid': False, - 'display': '1: foo', - 'redaction_plan': None, - 'step_name': 'Run inputs', - 'date': 'removed', - 'filename': None, - 'type': 'dataset', - 'id': 99, - 'size': 'removed'}, - {'is_ok': True, - 'errors': [], - 'name': '', - 'url': '/api/datasets/100/', - 'is_invalid': False, - 'display': '2: ', - 'redaction_plan': None, - 'step_name': '', - 'date': 'removed', - 'filename': None, - 'type': 'dataset', - 'id': 100, - 'size': 'removed'}], - 'id': 1234} - - data = RunOutputsSerializer(run).data - - self.maxDiff = None - self.assertEqual(expected_data['input_summary'], data['input_summary']) - self.assertEqual(expected_data, data) - - def test_run_outputs(self): - pipeline = Pipeline() - pipeline.outputs.add(TransformationOutput(dataset_idx=1, - dataset_name='foo'), - TransformationOutput(dataset_idx=2, - dataset_name='bar')) - for o in pipeline.outputs: - o.transformationoutput = o - run = Run(id=1234, pipeline=pipeline) - execrecord1 = ExecRecord() - execrecord1.execrecordouts.add(ExecRecordOut(dataset=Dataset())) - execrecord2 = ExecRecord() - execrecord2.execrecordouts.add(ExecRecordOut(dataset=Dataset())) - run.runoutputcables.add( - RunOutputCable(execrecord=execrecord1, - pipelineoutputcable=PipelineOutputCable(pipeline=pipeline, - output_name='foo')), - RunOutputCable(execrecord=execrecord2, - pipelineoutputcable=PipelineOutputCable(pipeline=pipeline, - output_name='bar'))) - expected_data = {'output_summary': [{'date': 'removed', - 'display': '1: foo', - 'errors': [], - 'filename': None, - 'id': None, - 'is_invalid': False, - 'is_ok': True, - 'name': 'foo', - 'redaction_plan': None, - 'size': 'removed', - 'step_name': 'Run outputs', - 'type': 'dataset', - 'url': None}, - {'date': 'removed', - 'display': '2: bar', - 'errors': [], - 'filename': None, - 'id': None, - 'is_invalid': False, - 'is_ok': True, - 'name': 'bar', - 'redaction_plan': None, - 'size': 'removed', - 'step_name': '', - 'type': 'dataset', - 'url': None}], - 'input_summary': [], - 'id': 1234} - - data = RunOutputsSerializer(run).data - - self.assertEqual(expected_data['output_summary'], data['output_summary']) - self.assertEqual(expected_data, data) - - @patch('archive.serializers.reverse', return_value='/some/url') - def test_step_outputs(self, mock_reverse): - pipeline = Pipeline() - run = Run(id=1234, pipeline=pipeline) - step = RunStep(pipelinestep=PipelineStep(step_num=1, - name='foo.py')) - step.log = ExecLog(start_time='11 May 2015', end_time='12 May 2015') - step.log.methodoutput = MethodOutput(id=99, - return_code=0, - output_log=ContentFile('Done.', - name='x'), - error_log=ContentFile('', - name='y')) - run.runsteps = MockSet(step) - expected_data = {'output_summary': [{'date': '12 May 2015', - 'display': 'Standard out', - 'errors': [], - 'filename': None, - 'id': 99, - 'is_invalid': False, - 'is_ok': True, - 'name': 'step_1_stdout', - 'redaction_plan': '/some/url', - 'size': u'5\xa0bytes', - 'step_name': '1: foo.py', - 'type': 'stdout', - 'url': '/some/url'}, - {'date': '12 May 2015', - 'display': 'Standard error', - 'errors': [], - 'filename': None, - 'id': 99, - 'is_invalid': False, - 'is_ok': True, - 'name': 'step_1_stderr', - 'redaction_plan': '/some/url', - 'size': u'0\xa0bytes', - 'step_name': '', - 'type': 'stderr', - 'url': '/some/url'}], - 'input_summary': [], - 'id': 1234} - - data = RunOutputsSerializer(run).data - self.maxDiff = None - self.assertEqual(expected_data['output_summary'], data['output_summary']) - self.assertEqual(expected_data, data) - - @patch('archive.serializers.reverse', return_value='/some/url') - def test_step_outputs_error(self, mock_reverse): - """ The return code is nonzero, but the stderr and stdout files exist. - """ - pipeline = Pipeline() - run = Run(id=1234, pipeline=pipeline) - step = RunStep(pipelinestep=PipelineStep(step_num=1, - name='foo.py')) - step.log = ExecLog(start_time='11 May 2015', end_time='12 May 2015') - step.log.methodoutput = MethodOutput(id=99, - return_code=1, - output_log=ContentFile('Done.', - name='x'), - error_log=ContentFile('Bad stuff.', - name='y')) - run.runsteps = MockSet(step) - expected_data = {'output_summary': [{'date': '12 May 2015', - 'display': 'Standard out', - 'errors': ['return code 1'], - 'filename': None, - 'id': 99, - 'is_invalid': False, - 'is_ok': True, - 'name': 'step_1_stdout', - 'redaction_plan': '/some/url', - 'size': u'5\xa0bytes', - 'step_name': '1: foo.py', - 'type': 'stdout', - 'url': '/some/url'}, - {'date': '12 May 2015', - 'display': 'Standard error', - 'errors': ['return code 1'], - 'filename': None, - 'id': 99, - 'is_invalid': False, - 'is_ok': True, - 'name': 'step_1_stderr', - 'redaction_plan': '/some/url', - 'size': u'10\xa0bytes', - 'step_name': '', - 'type': 'stderr', - 'url': '/some/url'}], - 'input_summary': [], - 'id': 1234} - - data = RunOutputsSerializer(run).data - self.maxDiff = None - self.assertEqual(expected_data['output_summary'], data['output_summary']) - self.assertEqual(expected_data, data) - - @patch('archive.serializers.reverse', return_value='/some/url') - def test_step_outputs_purged(self, mock_reverse): - """A runstep that ran successfully (return_code = 0), but whose - output file is missing (removed to save disk space) - - In this case, the MethodOutput fields for the file entries will be None, in contrast - to the test case below (files missing). - """ - pipeline = Pipeline() - run = Run(id=1234, pipeline=pipeline) - step = RunStep(pipelinestep=PipelineStep(step_num=1, - name='foo.py')) - step.log = ExecLog(start_time='11 May 2015', end_time='12 May 2015') - step.log.methodoutput = MethodOutput(id=99, return_code=0) - run.runsteps = MockSet(step) - expected_data = {'output_summary': [{'date': 'removed', - 'display': 'Standard out', - 'errors': [], - 'filename': None, - 'id': None, - 'is_invalid': False, - 'is_ok': True, - 'name': 'step_1_stdout', - 'redaction_plan': None, - 'size': 'removed', - 'step_name': '1: foo.py', - 'type': 'stdout', - 'url': None}, - {'date': 'removed', - 'display': 'Standard error', - 'errors': [], - 'filename': None, - 'id': None, - 'is_invalid': False, - 'is_ok': True, - 'name': 'step_1_stderr', - 'redaction_plan': None, - 'size': 'removed', - 'step_name': '', - 'type': 'stderr', - 'url': None}], - 'input_summary': [], - 'id': 1234} - - data = RunOutputsSerializer(run).data - self.maxDiff = None - self.assertEqual(expected_data['output_summary'], data['output_summary']) - self.assertEqual(expected_data, data) - - @patch('archive.serializers.reverse', return_value='/some/url') - def test_step_outputs_missing(self, mock_reverse): - """A runstep that ran successfully (return_code = 0), but whose - stdout and stderr file is missing due to some external error. - In this case, the MethodOutput fields for stdout and stderr must be nonzero on the db, - but the files must be missing on the file system. - """ - pipeline = Pipeline() - run = Run(id=1234, pipeline=pipeline) - step = RunStep(pipelinestep=PipelineStep(step_num=1, - name='foo.py')) - step.log = ExecLog(start_time='11 May 2015', end_time='12 May 2015') - OUTNAME = "foo.stdout" - ERRNAME = "foo.stderr" - step.log.methodoutput = MethodOutput(id=99, return_code=0, - output_log=OUTNAME, - error_log=ERRNAME) - # now the files exist in the db, but not in the file system. - run.runsteps = MockSet(step) - expected_data = {'output_summary': [{'date': 'missing', - 'display': 'Standard out', - 'errors': [], - 'filename': None, - 'id': None, - 'is_invalid': False, - 'is_ok': True, - 'name': 'step_1_stdout', - 'redaction_plan': None, - 'size': 'missing', - 'step_name': '1: foo.py', - 'type': 'stdout', - 'url': None}, - {'date': 'missing', - 'display': 'Standard error', - 'errors': [], - 'filename': None, - 'id': None, - 'is_invalid': False, - 'is_ok': True, - 'name': 'step_1_stderr', - 'redaction_plan': None, - 'size': 'missing', - 'step_name': '', - 'type': 'stderr', - 'url': None}], - 'input_summary': [], - 'id': 1234} - - data = RunOutputsSerializer(run).data - self.maxDiff = None - self.assertEqual(expected_data['output_summary'], data['output_summary']) - self.assertEqual(expected_data, data) - - @patch('archive.serializers.reverse', return_value='/some/url') - def test_step_outputs_running(self, mock_reverse): - pipeline = Pipeline() - run = Run(id=1234, pipeline=pipeline) - step = RunStep(pipelinestep=PipelineStep(step_num=1, - name='foo.py')) - step.log = ExecLog(start_time='11 May 2015') - step.log.methodoutput = MethodOutput(id=99) - run.runsteps = MockSet(step) - expected_data = {'output_summary': [{'date': None, - 'display': 'Running', - 'errors': [], - 'filename': None, - 'id': None, - 'is_invalid': False, - 'is_ok': False, - 'name': 'step_1_stdout', - 'redaction_plan': None, - 'size': None, - 'step_name': '1: foo.py', - 'type': 'stdout', - 'url': None}, - {'date': None, - 'display': 'Running', - 'errors': [], - 'filename': None, - 'id': None, - 'is_invalid': False, - 'is_ok': False, - 'name': 'step_1_stderr', - 'redaction_plan': None, - 'size': None, - 'step_name': '', - 'type': 'stderr', - 'url': None}], - 'input_summary': [], - 'id': 1234} - - data = RunOutputsSerializer(run).data - self.maxDiff = None - self.assertEqual(expected_data['output_summary'], data['output_summary']) - self.assertEqual(expected_data, data) - - -@mocked_relations(Run, Pipeline) -class QueuedRunMockTests(TestCase): - def create_run_with_runstep(self, runstep_state_id): - pipeline = Pipeline(family=PipelineFamily()) - pipelineoutputcable = PipelineOutputCable(pipelinecable_ptr_id=99, - pipeline=pipeline) - pipeline.outcables = MockSet(pipelineoutputcable) - user = User() - run = Run(pipeline=pipeline, user=user) - run.start(save=False) - run.runsteps = MockSet( - RunStep(_runcomponentstate_id=runstep_state_id, - pipelinestep=PipelineStep(step_num=1))) - run.runoutputcables = MockSet(RunOutputCable( - pipelineoutputcable=pipelineoutputcable)) - return run - - def test_owner(self): - expected_username = 'dave' - run = Run(user=User(username=expected_username)) - - progress = run.get_run_progress() - - self.assertSequenceEqual(expected_username, progress['user']) - - def test_unstarted_pipeline(self): - pipeline = Pipeline(family=PipelineFamily()) - user = User() - run = Run(pipeline=pipeline, user=user) - - progress = run.get_run_progress() - self.assertSequenceEqual("?", progress["status"]) - - def test_run_progress_empty_pipeline(self): - pipeline = Pipeline(family=PipelineFamily()) - user = User() - run = Run(pipeline=pipeline, user=user) - run.start(save=False) - - progress = run.get_run_progress() - - self.assertSequenceEqual('-', progress['status']) - - def test_run_progress_starting(self): - run = self.create_run_with_runstep(runcomponentstates.PENDING_PK) - - progress = run.get_run_progress() - - self.assertSequenceEqual('.-.', progress['status']) - - def test_run_progress_ready(self): - run = self.create_run_with_runstep(runcomponentstates.RUNNING_PK) - - progress = run.get_run_progress() - - self.assertSequenceEqual(':-.', progress['status']) - - def test_run_progress_started_steps(self): - run = self.create_run_with_runstep(runcomponentstates.RUNNING_PK) - run.runsteps.first().log = ExecLog() - - progress = run.get_run_progress() - - self.assertSequenceEqual('+-.', progress['status']) - - def test_run_progress_completed_steps(self): - run = self.create_run_with_runstep(runcomponentstates.SUCCESSFUL_PK) - - progress = run.get_run_progress() - - self.assertSequenceEqual('*-.', progress['status']) - - def test_run_progress_failed_steps(self): - run = self.create_run_with_runstep(runcomponentstates.FAILED_PK) - - progress = run.get_run_progress() - - self.assertSequenceEqual('!-.', progress['status']) - - def test_run_progress_output_ready(self): - run = self.create_run_with_runstep(runcomponentstates.SUCCESSFUL_PK) - run.runoutputcables.first()._runcomponentstate_id = runcomponentstates.RUNNING_PK - - progress = run.get_run_progress() - - self.assertSequenceEqual('*-:', progress['status']) - - def test_run_progress_output_running(self): - run = self.create_run_with_runstep(runcomponentstates.SUCCESSFUL_PK) - run.runoutputcables.first()._runcomponentstate_id = runcomponentstates.RUNNING_PK - run.runoutputcables.first().log = ExecLog() - - progress = run.get_run_progress() - - self.assertSequenceEqual('*-+', progress['status']) - - def test_run_progress_complete(self): - run = self.create_run_with_runstep(runcomponentstates.SUCCESSFUL_PK) - run.runoutputcables.first()._runcomponentstate_id = runcomponentstates.SUCCESSFUL_PK - - progress = run.get_run_progress() - - self.assertSequenceEqual('*-*', progress['status']) - - def test_run_progress_display_name_input(self): - run = self.create_run_with_runstep(runcomponentstates.PENDING_PK) - run.pipeline.family.name = 'Fasta2CSV' - run.inputs = MockSet(RunInput(dataset=Dataset( - name='TestFASTA', - dataset_file=ContentFile('contents', name='file1234.txt')))) - - progress = run.get_run_progress() - - self.assertSequenceEqual('Fasta2CSV on TestFASTA', progress['name']) - - def test_run_progress_display_name_date(self): - run = self.create_run_with_runstep(runcomponentstates.PENDING_PK) - run.pipeline.family.name = 'Fasta2CSV' - run.time_queued = '2010-02-14' - - progress = run.get_run_progress() - - self.assertSequenceEqual('Fasta2CSV at 2010-02-14', progress['name']) - - def test_run_progress_display_name_but_no_runsteps(self): - pipeline = Pipeline(family=PipelineFamily()) - user = User() - run = Run(pipeline=pipeline, user=user) - run.start(save=False) - run.pipeline.family.name = 'Fasta2CSV' - run.inputs = MockSet(RunInput(dataset=Dataset( - name='TestFASTA', - dataset_file=ContentFile('contents', name='file1234.txt')))) - - progress = run.get_run_progress() - - self.assertSequenceEqual('Fasta2CSV on TestFASTA', progress['name']) - - def test_run_progress_display_name_run_name_set(self): - expected_run_name = 'My custom run name' - run = self.create_run_with_runstep(runcomponentstates.PENDING_PK) - run.pipeline.family.name = 'Fasta2CSV' - run.time_queued = '2010-02-14' - run.name = expected_run_name - - progress = run.get_run_progress() - - self.assertSequenceEqual(expected_run_name, progress['name']) diff --git a/kive/archive/tests_queuedrun.py b/kive/archive/tests_queuedrun.py deleted file mode 100644 index 5b3d76672..000000000 --- a/kive/archive/tests_queuedrun.py +++ /dev/null @@ -1,1506 +0,0 @@ -from unittest import skipIf - -import os -import re -import tempfile -import itertools -import copy - -from mock import patch - -from django.conf import settings -from django.contrib.auth.models import User -from django.test import TestCase, skipIfDBFeature -from django.core.urlresolvers import reverse, resolve -from django.utils import timezone -from django.contrib.auth.models import Group - -from rest_framework.test import APIRequestFactory, force_authenticate -from rest_framework import status - -from archive.models import Run, ExecLog -from archive.serializers import RunSerializer, RunBatchSerializer, grplst2str, usrlst2str -from archive.exceptions import SandboxActiveException, RunNotFinished -from constants import runstates -from file_access_utils import create_sandbox_base_path -from fleet.dockerlib import SingularityDockerHandler -from librarian.models import ExecRecord, Dataset -from pipeline.models import Pipeline, PipelineFamily -from metadata.models import kive_user, everyone_group -from kive.testing_utils import clean_up_all_files -from kive.tests import install_fixture_files, remove_fixture_files, DuckContext,\ - BaseTestCases -from fleet.workers import Manager - - -@skipIfDBFeature('is_mocked') -class RemoveRedactRunInProgress(TestCase): - fixtures = ["em_sandbox_test_environment"] - - def setUp(self): - # Clear out all the Runs and ExecRecords in the environment. - # Run.objects.all().delete() - # ExecRecord.objects.all().delete() - - self.pf = PipelineFamily.objects.get(name="Pipeline_family") - self.myUser = self.pf.user - self.pE = self.pf.members.get(revision_name="pE_name") - self.triplet_dataset = Dataset.objects.filter(name="triplet").first() - self.doublet_dataset = Dataset.objects.get(name="doublet") - self.singlet_dataset = Dataset.objects.filter(name="singlet").first() - self.raw_dataset = Dataset.objects.get(name="raw_DS") - self.step_E1 = self.pE.steps.get(step_num=1) - self.mA = self.step_E1.transformation.definite - - # A run that's mid-progress. - self.run = Run(pipeline=self.pE, user=self.myUser) - self.run.save() - self.run.inputs.create( - index=1, - dataset=self.triplet_dataset - ) - self.run.inputs.create( - index=2, - dataset=self.singlet_dataset - ) - self.run.inputs.create( - index=3, - dataset=self.raw_dataset - ) - - self.rs_1 = self.run.runsteps.create( - pipelinestep=self.step_E1, - reused=False, - ) - self.rsic = self.rs_1.RSICs.create( - PSIC=self.step_E1.cables_in.first() - ) - self.rsic.log = ExecLog.create(self.rsic, self.rsic) - rsic_er = ExecRecord(generator=self.rsic.log) - rsic_er.save() - self.rsic.execrecord = rsic_er - self.rsic.save() - self.rsic.execrecord.execrecordins.create( - generic_input=self.pE.inputs.get(dataset_idx=3), - dataset=self.raw_dataset - ) - self.rsic.execrecord.execrecordouts.create( - generic_output=self.step_E1.transformation.definite.inputs.first(), - dataset=self.raw_dataset - ) - - self.rs_1_log = ExecLog.create(self.rs_1, self.rs_1) - self.rs_1_log.methodoutput.return_code = 0 - self.rs_1_log.methodoutput.save() - rs_1_er = ExecRecord(generator=self.rs_1_log) - rs_1_er.save() - self.rs_1.execrecord = rs_1_er - self.rs_1.save() - self.rs_1.execrecord.execrecordins.create( - generic_input=self.mA.inputs.first(), - dataset=self.raw_dataset - ) - self.rs_1.execrecord.execrecordouts.create( - generic_output=self.mA.outputs.first(), - dataset=self.doublet_dataset - ) - self.run.start(save=True) - - def tearDown(self): - clean_up_all_files() - - def test_remove_pipeline_fails(self): - """ - Removing the Pipeline of a Run that's in progress should fail. - """ - self.assertRaisesRegexp( - RunNotFinished, - "Cannot remove: an affected run is still in progress", - lambda: self.pE.remove() - ) - - def test_remove_dataset_fails(self): - """ - Removing a Dataset of a Run that's in progress should fail. - """ - self.assertRaisesRegexp( - RunNotFinished, - "Cannot remove: an affected run is still in progress", - lambda: self.triplet_dataset.remove() - ) - - def test_redact_dataset_fails(self): - """ - Redacting a Dataset of a Run that's in progress should fail. - """ - self.assertRaisesRegexp( - RunNotFinished, - "Cannot redact: an affected run is still in progress", - lambda: self.triplet_dataset.redact() - ) - - def test_remove_execrecord_fails(self): - """ - Removing an ExecRecord of a Run that's in progress should fail. - """ - self.assertRaisesRegexp( - RunNotFinished, - "Cannot remove: an affected run is still in progress", - lambda: self.rs_1.execrecord.remove() - ) - - def test_redact_execrecord_fails(self): - """ - Redacting an ExecRecord of a Run that's in progress should fail. - """ - self.assertRaisesRegexp( - RunNotFinished, - "Cannot redact: an affected run is still in progress", - lambda: self.rs_1.execrecord.redact() - ) - - -@skipIfDBFeature('is_mocked') -class RemoveRedactRunJustStarting(TestCase): - """ - Removal/redaction of stuff used in an unstarted run should be allowed. - """ - fixtures = ["em_sandbox_test_environment"] - - def setUp(self): - self.pf = PipelineFamily.objects.get(name="Pipeline_family") - self.myUser = self.pf.user - self.pE = self.pf.members.get(revision_name="pE_name") - self.triplet_dataset = Dataset.objects.filter(name="triplet").first() - self.doublet_dataset = Dataset.objects.get(name="doublet") - self.singlet_dataset = Dataset.objects.filter(name="singlet").first() - self.raw_dataset = Dataset.objects.get(name="raw_DS") - - # A run that's just starting, to the point that no Run exists yet. - self.run_just_starting = Run(user=self.myUser, pipeline=self.pE) - self.run_just_starting.save() - self.run_just_starting.inputs.create( - index=1, - dataset=self.triplet_dataset - ) - self.run_just_starting.inputs.create( - index=2, - dataset=self.singlet_dataset - ) - self.run_just_starting.inputs.create( - index=3, - dataset=self.raw_dataset - ) - - def tearDown(self): - clean_up_all_files() - - def test_remove_pipeline(self): - """ - Removing the Pipeline of an unstarted Run should work. - """ - self.pE.remove() - # self.assertRaisesRegexp( - # RunNotFinished, - # "Cannot remove: an affected run is still in progress", - # lambda: self.pE.remove() - # ) - - def test_remove_dataset(self): - """ - Removing a Dataset of an unstarted Run should work. - """ - self.triplet_dataset.remove() - # self.assertRaisesRegexp( - # RunNotFinished, - # "Cannot remove: an affected run is still in progress", - # lambda: self.triplet_dataset.remove() - # ) - - def test_redact_dataset(self): - """ - Redacting a Dataset of a Run should work. - """ - self.triplet_dataset.redact() - # self.assertRaisesRegexp( - # RunNotFinished, - # "Cannot redact: an affected run is still in progress", - # lambda: self.triplet_dataset.redact() - # ) - - -@skipIfDBFeature('is_mocked') -class RestoreReusableDatasetTest(BaseTestCases.SlurmExecutionTestCase): - """ - Scenario where an output is marked as reusable, and it needs to be restored. - - There are three methods: - * sums_and_products - take each row of two integers, calculate sum and - product, then shuffle all the result rows. This makes it reusable, but not - deterministic. - * total_sums - copy the first row, then one more row with the sum of all - the sums from the remaining rows. - * total_products - copy the first row, then one more row with the sum of all - the products from the remaining rows. - """ - fixtures = ["restore_reusable_dataset"] - - def setUp(self): - install_fixture_files("restore_reusable_dataset") - - def tearDown(self): - # remove_fixture_files() - pass - - def execute_pipeline(self, pipeline): - dataset = Dataset.objects.get(name="pairs") - mgr = Manager.execute_pipeline(pipeline.user, pipeline, [dataset]) - return mgr.get_last_run() - - def test_run_new_pipeline(self): - pipeline = Pipeline.objects.get(revision_name='sums and products') - run_to_process = self.execute_pipeline(pipeline) - self.assertIsNotNone(run_to_process) - if not run_to_process.is_successful(): - state_name = run_to_process.get_state_name() - print("unexpected run state name: '{}'".format(state_name)) - self.fail("run is not successful") - - @skipIf(not settings.RUN_SINGULARITY_TESTS, "Singularity tests disabled.") - def test_run_new_pipeline_with_singularity(self): - pipeline = Pipeline.objects.get(revision_name='sums and products') - dataset = Dataset.objects.get(name="pairs") - mgr = Manager.execute_pipeline( - pipeline.user, - pipeline, - [dataset], - singularity_handler_class=SingularityDockerHandler) - run_to_process = mgr.get_last_run() - self.assertIsNotNone(run_to_process) - if not run_to_process.is_successful(): - state_name = run_to_process.get_state_name() - print("unexpected run state name: '{}'".format(state_name)) - self.fail("run is not successful") - - def test_rerun_old_pipeline(self): - pipeline = Pipeline.objects.get(revision_name='sums only') - expected_execrecord_count = ExecRecord.objects.count() - - run_to_process = self.execute_pipeline(pipeline) - - self.assertTrue(run_to_process.is_successful()) - self.assertEqual(expected_execrecord_count, ExecRecord.objects.count()) - - -@skipIfDBFeature('is_mocked') -class GarbageCollectionTest(TestCase): - """ - Tests of sandbox garbage collection. - """ - fixtures = ["removal"] - - def setUp(self): - self.noop_pl = Pipeline.objects.filter( - family__name="Nucleotide Sequence Noop" - ).order_by( - "revision_number" - ).first() - - self.noop_run = Run.objects.filter( - pipeline=self.noop_pl - ).order_by( - "end_time" - ).first() - - # A phony directory that we mock-run a Pipeline in. - self.mock_sandbox_path = tempfile.mkdtemp( - prefix="user{}_run{}_".format(self.noop_run.user, self.noop_run.pk), - dir=create_sandbox_base_path()) - - def test_reap_nonexistent_sandbox_path(self): - """ A Run that has no sandbox path should do nothing. - - This used to raise an exception, but it's a valid scenario. When a run - is cancelled before it starts, it never gets a sandbox path. - """ - now = timezone.now() - run = Run(pipeline=self.noop_pl, - user=self.noop_pl.user, - start_time=now, - end_time=now, - _runstate_id=runstates.SUCCESSFUL_PK) - run.save() - - run.collect_garbage() - - reloaded_run = Run.objects.get(id=run.id) - self.assertTrue(reloaded_run.purged) - - def test_reap_unfinished_run(self): - """ - A Run that is not marked as finished should raise an exception. - """ - run = Run(pipeline=self.noop_pl, user=self.noop_pl.user, sandbox_path=self.mock_sandbox_path) - run.save() - - self.assertRaisesRegexp( - SandboxActiveException, - re.escape("Run (pk={}, Pipeline={}, queued {}, User=RemOver) is not finished".format( - run.id, - self.noop_pl, - run.time_queued)), - run.collect_garbage - ) - - def test_reap_finished_run(self): - """ - A Run that is finished should be reaped without issue. - """ - self.noop_run.sandbox_path = self.mock_sandbox_path - self.noop_run.save() - self.noop_run.collect_garbage() - - self.assertFalse(os.path.exists(self.mock_sandbox_path)) - self.assertTrue(self.noop_run.purged) - - def test_reap_on_removal(self): - """ - Removing a Run should reap the sandbox. - """ - self.noop_run.sandbox_path = self.mock_sandbox_path - self.noop_run.save() - - self.noop_run.remove() - self.assertFalse(os.path.exists(self.mock_sandbox_path)) - - def test_reap_on_redaction(self): - """ - Redacting part of a Run should reap the sandbox. - """ - self.noop_run.sandbox_path = self.mock_sandbox_path - self.noop_run.save() - - self.noop_run.runsteps.first().execrecord.redact() - self.assertFalse(os.path.exists(self.mock_sandbox_path)) - - -@skipIfDBFeature('is_mocked') -class RunApiTests(TestCase): - # This fixture has the result of sandbox.tests.execute_tests_environment_setup, - # as well of setting up another Pipeline; this other Pipeline and the resulting - # run is used in this test case. - fixtures = ["run_api_tests"] - - def setUp(self): - install_fixture_files("run_api_tests") - - self.kive_user = kive_user() - self.myUser = User.objects.get(username="john") - - self.pipeline_to_run = Pipeline.objects.get( - family__name="self.pf", - revision_name="pX_revision_2" - ) - - self.dataset = Dataset.objects.get( - name="pX_in_dataset", - structure__isnull=False, - user=self.myUser - ) - - self.factory = APIRequestFactory() - self.run_list_path = reverse('run-list') - self.run_list_view, _, _ = resolve(self.run_list_path) - self.run_status_path = reverse('run-status') - self.run_status_view, _, _ = resolve(self.run_status_path) - - def tearDown(self): - clean_up_all_files() - remove_fixture_files() - - def test_run_index(self, expected_runs=1): - request = self.factory.get(self.run_list_path) - response = self.run_list_view(request).render() - data = response.render().data - - self.assertEquals( - data['detail'], - "Authentication credentials were not provided.") - - force_authenticate(request, user=self.myUser) - response = self.run_list_view(request).render() - data = response.render().data - - self.assertEquals(len(data), expected_runs) - for run in data: - self.assertIn('id', run) - self.assertIn('removal_plan', run) - self.assertIn('run_status', run) - - def test_run_status(self): - expected_runs = 1 - request = self.factory.get(self.run_status_path) - force_authenticate(request, user=self.myUser) - response = self.run_status_view(request).render() - data = response.render().data - - self.assertEquals(len(data), expected_runs) - self.assertEqual('**-**', data[0]['run_progress']['status']) - - def test_pipeline_execute_plus_details_and_run_remove(self): - # TODO: This should be split into one test to test the pipeline execution - # Plus many tests to test details (which needs a proper fixture) - # Kick off the run - request = self.factory.post( - self.run_list_path, - { - "pipeline": self.pipeline_to_run.pk, - "inputs": [ - { - "index": 1, - "dataset": self.dataset.pk - } - ] - }, - format="json" - ) - force_authenticate(request, user=self.myUser) - response = self.run_list_view(request).render() - data = response.render().data - - # Check that the run created something sensible. - self.assertIn("id", data) - self.assertIn('run_outputs', data) - - # There should be two runs now: the one we just started (and hasn't run yet) - # and the one we already ran that's in the fixture. - self.test_run_index(2) - - # Let's examine a real execution. - real_run_pk = self.pipeline_to_run.pipeline_instances.first().pk - - # Touch the record detail page. - path = self.run_list_path + "{}/".format(real_run_pk) - request = self.factory.get(path) - force_authenticate(request, user=self.myUser) - view, args, kwargs = resolve(path) - response = view(request, *args, **kwargs) - data = response.render().data - - # Touch the run status page. - path = self.run_list_path + "{}/run_status/".format(real_run_pk) - request = self.factory.get(path) - force_authenticate(request, user=self.myUser) - view, args, kwargs = resolve(path) - response = view(request, *args, **kwargs) - data = response.render().data - self.assertEquals(data['status'], '**-**') - self.assertIn('step_progress', data) - - # Touch the outputs. - path = self.run_list_path + "{}/run_outputs/".format(real_run_pk) - request = self.factory.get(path) - force_authenticate(request, user=self.myUser) - view, args, kwargs = resolve(path) - response = view(request, *args, **kwargs) - data = response.render().data - self.assertEquals(data['id'], real_run_pk) - self.assertEquals(len(data['output_summary']), 8) - - for output in data['output_summary']: - self.assertEquals(output['is_ok'], True) - self.assertEquals(output['is_invalid'], False) - - # Touch the removal plan. - path = self.run_list_path + "{}/removal_plan/".format(real_run_pk) - request = self.factory.get(path) - force_authenticate(request, user=self.myUser) - view, args, kwargs = resolve(path) - response = view(request, *args, **kwargs) - data = response.render().data - - # 4 Datasets created: - # - 1 by the custom input cable to step 1 (and this is reused by the input cable to step 2) - # - 1 by step 1 - # - 1 by step 2 - # - 1 by the custom output cable - self.assertEquals(data['Datasets'], 4) - self.assertEquals(data['Runs'], 1) - self.assertEquals(data['Datatypes'], 0) - - # Delete the record. - path = self.run_list_path + "{}/".format(real_run_pk) - request = self.factory.delete(path) - force_authenticate(request, user=self.kive_user) - view, args, kwargs = resolve(path) - response = view(request, *args, **kwargs) - self.assertEquals(response.render().data, None) - self.test_run_index(1) # The run in the fixture should still be there. - - def test_stop_run(self): - """ - Test PATCHing a run to stop. - """ - request = self.factory.post( - self.run_list_path, - { - "pipeline": self.pipeline_to_run.pk, - "inputs": [ - { - "index": 1, - "dataset": self.dataset.pk - } - ] - }, - format="json" - ) - force_authenticate(request, user=self.myUser) - response = self.run_list_view(request).render() - data = response.render().data - - detail_path = reverse("run-detail", kwargs={'pk': data["id"]}) - request = self.factory.patch(detail_path, {'is_stop_requested': "true"}) - force_authenticate(request, user=self.myUser) - - detail_view, _, _ = resolve(detail_path) - response = detail_view(request, pk=data["id"]) - self.assertEquals(response.status_code, status.HTTP_200_OK) - - stopped_run = Run.objects.get(pk=data["id"]) - self.assertEquals(stopped_run.stopped_by, self.myUser) - - def test_stop_run_administrator(self): - """ - An administrator should be allowed to stop a run. - """ - request = self.factory.post( - self.run_list_path, - { - "pipeline": self.pipeline_to_run.pk, - "inputs": [ - { - "index": 1, - "dataset": self.dataset.pk - } - ] - }, - format="json" - ) - force_authenticate(request, user=self.myUser) - response = self.run_list_view(request).render() - data = response.render().data - - detail_path = reverse("run-detail", kwargs={'pk': data["id"]}) - request = self.factory.patch(detail_path, {'is_stop_requested': "true"}) - force_authenticate(request, user=self.kive_user) - - detail_view, _, _ = resolve(detail_path) - response = detail_view(request, pk=data["id"]) - self.assertEquals(response.status_code, status.HTTP_200_OK) - - stopped_run = Run.objects.get(pk=data["id"]) - self.assertEquals(stopped_run.stopped_by, self.kive_user) - - def test_stop_run_non_owner(self): - """ - A user who does not own the run should not be allowed to stop it. - """ - # First, we have to give other people access to the data and Pipeline. - self.dataset.grant_everyone_access() - - # This is only one layer deep so we don't have to recurse. - for xput in itertools.chain(self.pipeline_to_run.inputs.all(), self.pipeline_to_run.outputs.all()): - if xput.has_structure: - xput.structure.compounddatatype.grant_everyone_access() - - for step in self.pipeline_to_run.steps.all(): - curr_method = step.transformation.definite - - for xput in itertools.chain(curr_method.inputs.all(), curr_method.outputs.all()): - if xput.has_structure: - xput.structure.compounddatatype.grant_everyone_access() - - # Fortunately this has no dependencies. - curr_method.driver.coderesource.grant_everyone_access() - curr_method.driver.grant_everyone_access() - step.transformation.definite.family.grant_everyone_access() - step.transformation.grant_everyone_access() - - self.pipeline_to_run.family.grant_everyone_access() - self.pipeline_to_run.grant_everyone_access() - - request = self.factory.post( - self.run_list_path, - { - "pipeline": self.pipeline_to_run.pk, - "inputs": [ - { - "index": 1, - "dataset": self.dataset.pk - } - ], - "groups_allowed": ["Everyone"] - }, - format="json" - ) - force_authenticate(request, user=self.kive_user) - response = self.run_list_view(request).render() - data = response.render().data - - detail_path = reverse("run-detail", kwargs={'pk': data["id"]}) - request = self.factory.patch(detail_path, {'is_stop_requested': "true"}) - force_authenticate(request, user=self.myUser) - - detail_view, _, _ = resolve(detail_path) - response = detail_view(request, pk=data["id"]) - self.assertEquals(response.status_code, status.HTTP_403_FORBIDDEN) - - -@skipIfDBFeature('is_mocked') -class RunSerializerTestBase(TestCase): - fixtures = ["em_sandbox_test_environment"] - - def setUp(self): - install_fixture_files("em_sandbox_test_environment") - self.kive_user = kive_user() - self.myUser = User.objects.get(username="john") - - self.duck_context = DuckContext() - self.john_context = DuckContext(user=self.myUser) - - self.em_pf = PipelineFamily.objects.get(name="Pipeline_family") - self.em_pipeline = self.em_pf.members.get(revision_name="pE_name") - - # The inputs to this pipeline are (triplet_cdt, singlet_cdt, raw). - # The second one has min_row=10. - self.triplet_cdt = self.em_pipeline.inputs.get(dataset_idx=1).get_cdt() - self.singlet_cdt = self.em_pipeline.inputs.get(dataset_idx=2).get_cdt() - - # Datasets to feed the pipeline that are defined in the fixture. - self.triplet_SD = Dataset.objects.get( - name="triplet", - description="lol", - dataset_file__endswith="step_0_triplet.csv", - user=self.myUser, - structure__isnull=False, - structure__compounddatatype=self.triplet_cdt - ) - self.singlet_SD = Dataset.objects.get( - name="singlet", - description="lol", - dataset_file__endswith="singlet_cdt_large.csv", - user=self.myUser, - structure__isnull=False, - structure__compounddatatype=self.singlet_cdt - ) - self.raw_SD = Dataset.objects.get( - name="raw_DS", - description="lol", - user=self.myUser - ) - - def tearDown(self): - remove_fixture_files() - - -class RunSerializerTests(RunSerializerTestBase): - def test_validate(self): - """ - Validating a well-specified Run to process. - """ - serialized_rtp = { - "pipeline": self.em_pipeline.pk, - "inputs": [ - { - "dataset": self.triplet_SD.pk, - "index": 1 - }, - { - "dataset": self.singlet_SD.pk, - "index": 2 - }, - { - "dataset": self.raw_SD.pk, - "index": 3 - } - ], - "users_allowed": [], - "groups_allowed": [] - } - rtp_serializer = RunSerializer(data=serialized_rtp, context=self.john_context) - - self.assertTrue(rtp_serializer.is_valid()) - - def test_validate_wrong_number_inputs(self): - """ - Validation fails if the number of inputs is wrong. - """ - serialized_rtp = { - "pipeline": self.em_pipeline.pk, - "inputs": [ - { - "dataset": self.triplet_SD.pk, - "index": 1 - }, - { - "dataset": self.singlet_SD.pk, - "index": 2 - } - ], - "users_allowed": [], - "groups_allowed": [] - } - rtp_serializer = RunSerializer(data=serialized_rtp, context=self.john_context) - - self.assertFalse(rtp_serializer.is_valid()) - self.assertEquals([str(e) for e in rtp_serializer.errors["non_field_errors"]], - [u"Pipeline has 3 inputs, but only received 2."]) - - def test_validate_inputs_oversated(self): - """ - Validation fails if an input has more than one input defined. - """ - serialized_rtp = { - "pipeline": self.em_pipeline.pk, - "inputs": [ - { - "dataset": self.triplet_SD.pk, - "index": 1 - }, - { - "dataset": self.singlet_SD.pk, - "index": 2 - }, - { - "dataset": self.triplet_SD.pk, - "index": 1 - }, - ], - "users_allowed": [], - "groups_allowed": [] - } - rtp_serializer = RunSerializer(data=serialized_rtp, context=self.john_context) - - self.assertFalse(rtp_serializer.is_valid()) - self.assertEquals([str(e) for e in rtp_serializer.errors["non_field_errors"]], - [u"Pipeline inputs must be uniquely specified"]) - - def test_validate_input_index_dne(self): - """ - Validation fails if an input index doesn't exist. - """ - serialized_rtp = { - "pipeline": self.em_pipeline.pk, - "inputs": [ - { - "dataset": self.triplet_SD.pk, - "index": 1 - }, - { - "dataset": self.singlet_SD.pk, - "index": 2 - }, - { - "dataset": self.triplet_SD.pk, - "index": 4 - }, - ], - "users_allowed": [], - "groups_allowed": [] - } - rtp_serializer = RunSerializer(data=serialized_rtp, context=self.john_context) - - self.assertFalse(rtp_serializer.is_valid()) - self.assertEquals([str(e) for e in rtp_serializer.errors["non_field_errors"]], - [u"Pipeline {} has no input with index {}".format( - self.em_pipeline, 4 - )]) - - def test_validate_illegal_priority(self): - """ - Validation fails if the job's priority is out of bounds (normally 0..2) - """ - serialized_rtp = { - "pipeline": self.em_pipeline.pk, - "inputs": [ - { - "dataset": self.triplet_SD.pk, - "index": 1 - }, - { - "dataset": self.singlet_SD.pk, - "index": 2 - }, - { - "dataset": self.triplet_SD.pk, - "index": 4 - }, - ], - "users_allowed": [], - "groups_allowed": [], - "priority": 1001 - } - rtp_serializer = RunSerializer(data=serialized_rtp, context=self.john_context) - - self.assertFalse(rtp_serializer.is_valid()) - self.assertEquals([str(e) for e in rtp_serializer.errors["non_field_errors"]], - [u"Illegal priority level"]) - - def test_validate_input_CDT_incompatible(self): - """ - Validation fails if an input Dataset is incompatible with the Pipeline input. - """ - serialized_rtp = { - "pipeline": self.em_pipeline.pk, - "inputs": [ - { - "dataset": self.triplet_SD.pk, - "index": 1 - }, - { - "dataset": self.singlet_SD.pk, - "index": 2 - }, - { - "dataset": self.triplet_SD.pk, - "index": 3 - }, - ], - "users_allowed": [], - "groups_allowed": [] - } - rtp_serializer = RunSerializer(data=serialized_rtp, context=self.john_context) - - self.assertFalse(rtp_serializer.is_valid()) - self.assertEquals(rtp_serializer.errors["non_field_errors"], - [u"Input {} is incompatible with Dataset {}".format( - self.em_pipeline.inputs.get(dataset_idx=3), self.triplet_SD - )]) - - def test_validate_overextending_permissions(self): - """ - Validation fails if users_allowed and groups_allowed exceed those on the inputs and the Pipeline. - """ - self.em_pipeline.groups_allowed.remove(everyone_group()) - serialized_rtp = { - "pipeline": self.em_pipeline.pk, - "inputs": [ - { - "dataset": self.triplet_SD.pk, - "index": 1 - }, - { - "dataset": self.singlet_SD.pk, - "index": 2 - }, - { - "dataset": self.raw_SD.pk, - "index": 3 - } - ], - "users_allowed": [self.kive_user], - "groups_allowed": [everyone_group()] - } - rtp_serializer = RunSerializer(data=serialized_rtp, context=self.john_context) - - self.assertFalse(rtp_serializer.is_valid()) - self.assertEqual( - set([str(e) for e in rtp_serializer.errors["non_field_errors"]]), - set([ - u"User(s) {} may not be granted access".format(usrlst2str([self.kive_user])), - u"Group(s) {} may not be granted access".format(grplst2str([everyone_group()])) - ]) - ) - - def test_create(self): - """ - Creating a Run to process, i.e. adding a job to the queue. - """ - serialized_rtp = { - "pipeline": self.em_pipeline.pk, - "inputs": [ - { - "dataset": self.triplet_SD.pk, - "index": 1 - }, - { - "dataset": self.singlet_SD.pk, - "index": 2 - }, - { - "dataset": self.raw_SD.pk, - "index": 3 - } - ], - "users_allowed": [kive_user()], - "groups_allowed": [everyone_group()] - } - - rtp_serializer = RunSerializer(data=serialized_rtp, context=self.john_context) - self.assertTrue(rtp_serializer.is_valid()) - - before = timezone.now() - rtp = rtp_serializer.save() - after = timezone.now() - - # Probe the RunToProcess to check that it was correctly created. - self.assertEqual(rtp.pipeline, self.em_pipeline) - self.assertEqual(rtp.user, self.myUser) - self.assertEqual(set(rtp.users_allowed.all()), set([kive_user()])) - self.assertEqual(set(rtp.groups_allowed.all()), set([everyone_group()])) - self.assertTrue(before <= rtp.time_queued) - self.assertTrue(after >= rtp.time_queued) - - self.assertEqual(rtp.inputs.count(), 3) - self.assertEqual(rtp.inputs.get(index=1).dataset, self.triplet_SD) - self.assertEqual(rtp.inputs.get(index=2).dataset, self.singlet_SD) - self.assertEqual(rtp.inputs.get(index=3).dataset, self.raw_SD) - - -class RunBatchSerializerTests(RunSerializerTestBase): - def setUp(self): - super(RunBatchSerializerTests, self).setUp() - self.serialized_run = { - "pipeline": self.em_pipeline.pk, - "inputs": [ - { - "dataset": self.triplet_SD.pk, - "index": 1 - }, - { - "dataset": self.singlet_SD.pk, - "index": 2 - }, - { - "dataset": self.raw_SD.pk, - "index": 3 - } - ], - } - - def test_validate(self): - """ - Validating a well-formed RunBatch. - """ - serialized_rb = { - "name": "My RunBatch", - "description": "foo", - "runs": [ - self.serialized_run, - self.serialized_run, - self.serialized_run - ] - } - rb_serializer = RunBatchSerializer(data=serialized_rb, context=self.john_context) - self.assertTrue(rb_serializer.is_valid()) - - def test_validate_bad_permissions(self): - """ - Validation fails if the Runs' permissions exceed those on the RunBatch. - """ - second_serialized_run = copy.deepcopy(self.serialized_run) - - self.serialized_run["users_allowed"] = [self.kive_user.username] - self.serialized_run["groups_allowed"] = [everyone_group().name] - self.serialized_run["name"] = "OverlyPermissiveRun" - - second_serialized_run["users_allowed"] = [self.kive_user.username] - second_serialized_run["groups_allowed"] = [] - second_serialized_run["name"] = "LessPermissiveButStillTooMuchRun" - - serialized_rb = { - "name": "RunBatch with permissions issues", - "description": "foo", - "runs": [ - self.serialized_run, - second_serialized_run - ] - } - - rb_serializer = RunBatchSerializer(data=serialized_rb, context=self.john_context) - self.assertFalse(rb_serializer.is_valid()) - # NOTE: use sets here, as the order of the error messages is unimportant - expected_set = set([ - "Group(s) {} may not be granted access to run {} (index {})".format( - grplst2str([everyone_group()]), - self.serialized_run["name"], - 1 - ), - "User(s) {} may not be granted access to run {} (index {})".format( - usrlst2str([self.kive_user]), - self.serialized_run["name"], - 1 - ), - "User(s) {} may not be granted access to run {} (index {})".format( - usrlst2str([self.kive_user]), - second_serialized_run["name"], - 2 - ), - ]) - # we must convert the list of ErrorDetail into strings first... - got_set = set([str(e) for e in rb_serializer.errors["non_field_errors"]]) - self.assertSetEqual(expected_set, got_set) - # assert False, "force fail" - - def test_validate_coherent_permissions(self): - """ - Validating a well-formed RunBatch with coherent permissions. - """ - second_serialized_run = copy.deepcopy(self.serialized_run) - - self.serialized_run["users_allowed"] = [self.kive_user.username] - self.serialized_run["name"] = "LetKiveSee" - - second_serialized_run["users_allowed"] = [self.kive_user.username] - second_serialized_run["groups_allowed"] = [] - second_serialized_run["name"] = "LetKiveSeeToo" - - serialized_rb = { - "name": "RunBatch that Kive sees", - "description": "foo", - "runs": [ - self.serialized_run, - second_serialized_run - ], - "users_allowed": [self.kive_user.username] - } - - rb_serializer = RunBatchSerializer(data=serialized_rb, context=self.john_context) - self.assertTrue(rb_serializer.is_valid()) - - def test_validate_batch_permissions_not_allowed(self): - """ - Validating a RunBatch whose permissions exceed those allowed by its Runs. - """ - self.em_pipeline.groups_allowed.remove(everyone_group()) - - self.serialized_run["name"] = "Run whose pipeline doesn't allow everyone to use it" - - serialized_rb = { - "name": "RunBatch that Kive and everyone sees", - "description": "foo", - "runs": [ - self.serialized_run, - ], - "users_allowed": [self.kive_user.username], - "groups_allowed": [everyone_group().name] - } - - rb_serializer = RunBatchSerializer(data=serialized_rb, context=self.john_context) - self.assertFalse(rb_serializer.is_valid()) - expected_set = set([ - "User(s) {} may not be granted access to run {} (index {})".format( - usrlst2str([self.kive_user]), - self.serialized_run["name"], - 1 - ), - "Group(s) {} may not be granted access to run {} (index {})".format( - grplst2str([everyone_group()]), - self.serialized_run["name"], - 1 - ) - ]) - got_set = set([str(e) for e in rb_serializer.errors["non_field_errors"]]) - self.assertSetEqual(expected_set, got_set) - - def test_validate_everyone_has_access(self): - """ - Validating a well-formed RunBatch. - """ - second_serialized_run = copy.deepcopy(self.serialized_run) - third_serialized_run = copy.deepcopy(self.serialized_run) - - self.serialized_run["users_allowed"] = [self.kive_user.username] - self.serialized_run["name"] = "LetKiveSee" - - second_serialized_run["users_allowed"] = [self.kive_user.username] - second_serialized_run["groups_allowed"] = [everyone_group().name] - second_serialized_run["name"] = "EveryoneSees" - - third_serialized_run["name"] = "OwnerOnly" - - serialized_rb = { - "name": "RunBatch that everyone sees", - "description": "foo", - "runs": [ - self.serialized_run, - second_serialized_run - ], - "groups_allowed": [everyone_group().name] - } - - rb_serializer = RunBatchSerializer(data=serialized_rb, context=self.john_context) - self.assertTrue(rb_serializer.is_valid()) - - def test_create(self): - """ - Create a well-formed RunBatch. - """ - second_serialized_run = copy.deepcopy(self.serialized_run) - third_serialized_run = copy.deepcopy(self.serialized_run) - - self.serialized_run["name"] = "One" - - second_serialized_run["name"] = "Two" - second_serialized_run["users_allowed"] = [self.kive_user.username] - - third_serialized_run["name"] = "Three" - third_serialized_run["groups_allowed"] = [everyone_group().name] - - serialized_rb = { - "name": "My RunBatch", - "description": "foo", - "runs": [ - self.serialized_run, - second_serialized_run, - third_serialized_run - ], - "users_allowed": [self.kive_user.username], - "groups_allowed": [everyone_group().name] - } - rb_serializer = RunBatchSerializer(data=serialized_rb, context=self.john_context) - self.assertTrue(rb_serializer.is_valid()) - rb = rb_serializer.save() - - # Probe the RunBatch to check that it was correctly created. - self.assertEqual(rb.user, self.myUser) - self.assertEqual(set(rb.users_allowed.all()), set([self.kive_user])) - self.assertEqual(set(rb.groups_allowed.all()), set([everyone_group()])) - - self.assertEqual(rb.runs.count(), 3) - - # Run One inherits its permissions. - run1 = rb.runs.get(name="One") - self.assertEqual(run1.users_allowed.count(), 1) - self.assertEqual(run1.users_allowed.first(), self.kive_user) - self.assertEqual(run1.groups_allowed.count(), 1) - self.assertEqual(run1.groups_allowed.first(), everyone_group()) - - # Runs Two and Three had their own permissions defined. - run2 = rb.runs.get(name="Two") - self.assertEqual(run2.users_allowed.count(), 1) - self.assertEqual(run2.users_allowed.first(), self.kive_user) - self.assertEqual(run2.groups_allowed.count(), 0) - - run3 = rb.runs.get(name="Three") - self.assertEqual(run3.users_allowed.count(), 0) - self.assertEqual(run3.groups_allowed.count(), 1) - self.assertEqual(run3.groups_allowed.first(), everyone_group()) - - def test_create_no_permissions_copied(self): - """ - Create a well-formed RunBatch without copying permissions over. - """ - second_serialized_run = copy.deepcopy(self.serialized_run) - third_serialized_run = copy.deepcopy(self.serialized_run) - - self.serialized_run["name"] = "One" - - second_serialized_run["name"] = "Two" - second_serialized_run["users_allowed"] = [self.kive_user.username] - - third_serialized_run["name"] = "Three" - third_serialized_run["groups_allowed"] = [everyone_group().name] - - serialized_rb = { - "name": "My RunBatch", - "description": "foo", - "runs": [ - self.serialized_run, - second_serialized_run, - third_serialized_run - ], - "users_allowed": [self.kive_user.username], - "groups_allowed": [everyone_group().name], - "copy_permissions_to_runs": False - } - rb_serializer = RunBatchSerializer(data=serialized_rb, context=self.john_context) - self.assertTrue(rb_serializer.is_valid()) - rb = rb_serializer.save() - - # Probe the RunBatch to check that it was correctly created. - self.assertEqual(rb.user, self.myUser) - self.assertEqual(set(rb.users_allowed.all()), set([self.kive_user])) - self.assertEqual(set(rb.groups_allowed.all()), set([everyone_group()])) - - self.assertEqual(rb.runs.count(), 3) - - # Run One does not inherit any permissions. - run1 = rb.runs.get(name="One") - self.assertEqual(run1.users_allowed.count(), 0) - self.assertEqual(run1.groups_allowed.count(), 0) - - # Runs Two and Three had their own permissions defined. - run2 = rb.runs.get(name="Two") - self.assertEqual(run2.users_allowed.count(), 1) - self.assertEqual(run2.users_allowed.first(), self.kive_user) - self.assertEqual(run2.groups_allowed.count(), 0) - - run3 = rb.runs.get(name="Three") - self.assertEqual(run3.users_allowed.count(), 0) - self.assertEqual(run3.groups_allowed.count(), 1) - self.assertEqual(run3.groups_allowed.first(), everyone_group()) - - def test_validate_update_good(self): - """ - Validating a good update to a RunBatch. - """ - # First, we create a RunBatch as we did in the above. - second_serialized_run = copy.deepcopy(self.serialized_run) - third_serialized_run = copy.deepcopy(self.serialized_run) - - self.serialized_run["name"] = "One" - - second_serialized_run["name"] = "Two" - second_serialized_run["users_allowed"] = [self.kive_user.username] - - third_serialized_run["name"] = "Three" - third_serialized_run["groups_allowed"] = [everyone_group().name] - - serialized_rb = { - "name": "My RunBatch", - "description": "foo", - "runs": [ - self.serialized_run, - second_serialized_run, - third_serialized_run - ], - "users_allowed": [self.kive_user.username], - "groups_allowed": [everyone_group().name], - "copy_permissions_to_runs": False - } - rb_serializer = RunBatchSerializer(data=serialized_rb, context=self.john_context) - self.assertTrue(rb_serializer.is_valid()) - - @patch.object(Run, "is_complete", return_value=True) - def test_validate_update_copy_permissions_invalid(self, _): - """ - Validating an update to a RunBatch which attempts to copy permissions but the permissions are not coherent. - """ - self.em_pipeline.groups_allowed.remove(everyone_group()) - # Create a RunBatch with appropriate permissions. - second_serialized_run = copy.deepcopy(self.serialized_run) - third_serialized_run = copy.deepcopy(self.serialized_run) - - self.serialized_run["name"] = "One" - - second_serialized_run["name"] = "Two" - - third_serialized_run["name"] = "Three" - - serialized_rb = { - "name": "My RunBatch", - "description": "foo", - "runs": [ - self.serialized_run, - second_serialized_run, - third_serialized_run - ], - "copy_permissions_to_runs": False - } - rb_serializer = RunBatchSerializer(data=serialized_rb, context=self.john_context) - self.assertTrue(rb_serializer.is_valid()) - rb = rb_serializer.save() - - run1 = rb.runs.get(name="One") - run2 = rb.runs.get(name="Two") - run3 = rb.runs.get(name="Three") - - # Now we attempt to update it, adding a permission that won't work. - new_group = Group(name="Interlopers") - new_group.save() - update_dict = { - "name": "My updated RunBatch", - "groups_allowed": [new_group.name] - } - update_serializer = RunBatchSerializer(rb, data=update_dict, context=self.john_context) - self.assertFalse(update_serializer.is_valid()) # note that we patched Run.is_complete so this would work - - self.assertSetEqual( - set([str(e) for e in update_serializer.errors["non_field_errors"]]), - { - "Group(s) {} may not be granted access to run {}".format( - grplst2str([new_group]), - run1 - ), - "Group(s) {} may not be granted access to run {}".format( - grplst2str([new_group]), - run2, - ), - "Group(s) {} may not be granted access to run {}".format( - grplst2str([new_group]), - run3 - ) - } - ) - - @patch.object(Run, "is_complete", return_value=True) - def test_validate_update_no_copy_permissions(self, _): - """ - Validating an update to a RunBatch which does not copy permissions. - """ - self.em_pipeline.groups_allowed.remove(everyone_group()) - # Create a RunBatch with appropriate permissions. - second_serialized_run = copy.deepcopy(self.serialized_run) - third_serialized_run = copy.deepcopy(self.serialized_run) - - self.serialized_run["name"] = "One" - - second_serialized_run["name"] = "Two" - - third_serialized_run["name"] = "Three" - - serialized_rb = { - "name": "My RunBatch", - "description": "foo", - "runs": [ - self.serialized_run, - second_serialized_run, - third_serialized_run - ], - "copy_permissions_to_runs": False - } - rb_serializer = RunBatchSerializer(data=serialized_rb, context=self.john_context) - self.assertTrue(rb_serializer.is_valid()) - rb = rb_serializer.save() - - # Now we attempt to update it, adding a permission that won't work for the runs, - # but it doesn't matter because we aren't changing anything. - new_group = Group(name="Interlopers") - new_group.save() - update_dict = { - "name": "My updated RunBatch", - "groups_allowed": [new_group.name], - "copy_permissions_to_runs": False - } - update_serializer = RunBatchSerializer(rb, data=update_dict, context=self.john_context) - self.assertTrue(update_serializer.is_valid()) # note that we patched Run.is_complete so this would work - - @patch.object(Run, "is_complete", return_value=True) - def test_update_copy_permissions(self, _): - """ - Update a RunBatch, copying permissions. - """ - # First, we create a RunBatch as we did in the above. - second_serialized_run = copy.deepcopy(self.serialized_run) - third_serialized_run = copy.deepcopy(self.serialized_run) - - self.serialized_run["name"] = "One" - - second_serialized_run["name"] = "Two" - second_serialized_run["users_allowed"] = [self.kive_user.username] - - third_serialized_run["name"] = "Three" - third_serialized_run["groups_allowed"] = [everyone_group().name] - - serialized_rb = { - "name": "My RunBatch", - "description": "foo", - "runs": [ - self.serialized_run, - second_serialized_run, - third_serialized_run - ], - "users_allowed": [self.kive_user.username], - "groups_allowed": [everyone_group().name], - "copy_permissions_to_runs": False - } - rb_serializer = RunBatchSerializer(data=serialized_rb, context=self.john_context) - self.assertTrue(rb_serializer.is_valid()) - rb = rb_serializer.save() - - # Now we update the RunBatch. - new_group = Group(name="Interlopers") - new_group.save() - update_dict = { - "name": "My updated RunBatch", - "groups_allowed": [new_group.name] - } - update_serializer = RunBatchSerializer(rb, data=update_dict, context=self.john_context) - update_serializer.is_valid() # note that we patched Run.is_complete - update_serializer.save() - - # Probe the RunBatch to check that it was correctly updated. - self.assertEqual(rb.user, self.myUser) - self.assertEqual(rb.name, update_dict["name"]) - self.assertSetEqual(set(rb.users_allowed.all()), {self.kive_user}) - self.assertEqual(set(rb.groups_allowed.all()), set([everyone_group(), new_group])) - - self.assertEqual(rb.runs.count(), 3) - - # All runs had the Everyone group added. - run1 = rb.runs.get(name="One") - self.assertFalse(run1.users_allowed.exists()) - self.assertSetEqual(set(run1.groups_allowed.all()), {new_group}) - - run2 = rb.runs.get(name="Two") - self.assertSetEqual(set(run2.users_allowed.all()), {self.kive_user}) - self.assertSetEqual(set(run2.groups_allowed.all()), {new_group}) - - run3 = rb.runs.get(name="Three") - self.assertFalse(run3.users_allowed.exists()) - self.assertSetEqual(set(run3.groups_allowed.all()), {everyone_group(), new_group}) - - @patch.object(Run, "is_complete", return_value=True) - def test_update_no_copy_permissions(self, _): - """ - Update a RunBatch, not copying permissions. - """ - # First, we create a RunBatch as we did in the above. - second_serialized_run = copy.deepcopy(self.serialized_run) - third_serialized_run = copy.deepcopy(self.serialized_run) - - self.serialized_run["name"] = "One" - - second_serialized_run["name"] = "Two" - second_serialized_run["users_allowed"] = [self.kive_user.username] - - third_serialized_run["name"] = "Three" - - serialized_rb = { - "name": "My RunBatch", - "description": "foo", - "runs": [ - self.serialized_run, - second_serialized_run, - third_serialized_run - ], - "users_allowed": [self.kive_user.username], - "copy_permissions_to_runs": False - } - rb_serializer = RunBatchSerializer(data=serialized_rb, context=self.john_context) - self.assertTrue(rb_serializer.is_valid()) - rb = rb_serializer.save() - - # Now we update the RunBatch. This newly-added group won't be propagated to the runs. - new_group = Group(name="Interlopers") - new_group.save() - update_dict = { - "name": "My updated RunBatch", - "groups_allowed": [new_group.name], - "copy_permissions_to_runs": False - } - update_serializer = RunBatchSerializer(rb, data=update_dict, context=self.john_context) - update_serializer.is_valid() # note that we patched Run.is_complete - update_serializer.save() - - # Probe the RunBatch to check that it was correctly updated. - self.assertEqual(rb.user, self.myUser) - self.assertEqual(rb.name, update_dict["name"]) - self.assertSetEqual(set(rb.users_allowed.all()), {self.kive_user}) - self.assertEqual(set(rb.groups_allowed.all()), {new_group}) - - self.assertEqual(rb.runs.count(), 3) - - # All runs had the Everyone group added. - run1 = rb.runs.get(name="One") - self.assertFalse(run1.users_allowed.exists()) - self.assertFalse(run1.groups_allowed.exists()) - - run2 = rb.runs.get(name="Two") - self.assertSetEqual(set(run2.users_allowed.all()), {self.kive_user}) - self.assertFalse(run2.groups_allowed.exists()) - - run3 = rb.runs.get(name="Three") - self.assertFalse(run3.users_allowed.exists()) - self.assertFalse(run3.groups_allowed.exists()) diff --git a/kive/container/management/commands/consolidate_containers.py b/kive/container/management/commands/consolidate_containers.py deleted file mode 100644 index c946e8c45..000000000 --- a/kive/container/management/commands/consolidate_containers.py +++ /dev/null @@ -1,76 +0,0 @@ -from __future__ import print_function -import logging -import os -import sys -from argparse import ArgumentDefaultsHelpFormatter - -from django.core.management.base import BaseCommand -from django.core.files import File -from django.conf import settings - -from container.models import Container - - -logger = logging.getLogger(__name__) - - -# This assumes we are using the default FileSystemStorage class. -class Command(BaseCommand): - help = "Consolidates Container file locations into their default directory." - - container_directory = os.path.normpath(os.path.join(settings.MEDIA_ROOT, Container.UPLOAD_DIR)) - - def add_arguments(self, parser): - parser.formatter_class = ArgumentDefaultsHelpFormatter - - def handle(self, **options): - containers_to_move = self.identify_containers_to_move() - prompt_message = "The following Container files will be moved to {}:\n".format(self.container_directory) - for container in containers_to_move: - prompt_message += "{} ({})\n".format(container, container.file.path) - prompt_message += "Proceed?" - proceed = self.confirm(prompt_message) - if not proceed: - return - self.move_container_files(containers_to_move) - - def identify_containers_to_move(self): - """ - Identify Containers whose files must move. - :return: - """ - containers_to_consolidate = [] - # Scan through all Containers and examine the filenames. - for container in Container.objects.all(): - current_absolute_path = os.path.normpath(container.file.path) - directory_in_storage = os.path.dirname(current_absolute_path) - if directory_in_storage == self.container_directory: - continue - containers_to_consolidate.append(container) - - return containers_to_consolidate - - def move_container_files(self, containers_to_move): - """ - Move Container files around as specified. - - :param containers_to_move: a list of Containers - :return: - """ - for container in containers_to_move: - print("Moving {} to {}....".format(container.file.path, self.container_directory)) - - original_path = container.file.path - new_name = os.path.basename(container.file.name) - with open(original_path, "rb") as f: - container.file.save(new_name, File(f), save=True) - - os.remove(original_path) - - # This is copied from the "purge.py" management command. - @staticmethod - def confirm(self, prompt): - print(prompt, end=' ') - confirmation = sys.stdin.readline().strip() - is_confirmed = confirmation.lower() == 'y' - return is_confirmed diff --git a/kive/container/management/commands/convert_docker_images.py b/kive/container/management/commands/convert_docker_images.py deleted file mode 100644 index 076ac85fd..000000000 --- a/kive/container/management/commands/convert_docker_images.py +++ /dev/null @@ -1,115 +0,0 @@ -from __future__ import print_function -import logging -import sys - -from django.core.management.base import BaseCommand - -from archive.models import RunStep -from container.models import Container -from method.models import Method, DockerImage - - -logger = logging.getLogger(__name__) - - -class Command(BaseCommand): - help = "Converts methods from using docker to singularity." - - def handle(self, *args, **options): - # See also: utils/request_reruns.py - print('Containers with descriptions that start with "Broken:":') - broken_containers = Container.objects.filter( - description__istartswith='Broken:') - methods = Method.objects.filter( - container__in=broken_containers).order_by('container__family__name', - 'container__tag', - 'family__name', - 'revision_number') - for method in methods: - print(method, - 'uses', - method.container.family, - method.container.tag) - if not methods: - print('None in use.') - else: - if self.confirm("Set these methods' containers to NULL? [y/N]"): - for method in methods: - method.container = None - method.save() - print('Done.') - print() - - print('Docker images needing to convert:') - images_to_convert = DockerImage.objects.filter( - pk__in=Method.objects.filter( - docker_image_id__isnull=False, - container_id__isnull=True).values('docker_image_id')) - - conversions = [] # [(image, container)] - for image in images_to_convert: - try: - container = Container.objects.exclude( - pk__in=broken_containers.values('pk')).get( - family__git=image.git, - tag__startswith=image.tag + '-singularity') - except Container.DoesNotExist: - latest_run_step = self.find_run_step(image) - print(image.name, - image.git, - image.tag, - 'run', - latest_run_step and latest_run_step.run_id) - continue - conversions.append((image, container)) - - print() - print('Docker images to convert:') - for image, container in conversions: - latest_run_step = self.find_run_step(image) - print(image.name, - image.git, - image.tag, - '=>', - container.tag, - 'run', - latest_run_step and latest_run_step.run_id) - - if not conversions: - print('None found.') - return - elif not self.confirm('Continue? [y/N]'): - return - logger.info('Starting.') - for image, container in conversions: - for method in image.methods.all(): - if method.container is not None: - logger.info('skipping %s.', method) - else: - try: - latest_run_step = self.find_method_run_step(method) - run_text = 'run {}'.format(latest_run_step.run_id) - except RunStep.DoesNotExist: - run_text = 'unused' - - method.container = container - method.save() - logger.info('converted %s (%s).', method, run_text) - logger.info('Done.') - - def find_run_step(self, image): - latest_run_step = RunStep.objects.filter( - pipelinestep__transformation__method__docker_image_id=image.id).order_by( - 'run_id').last() - return latest_run_step - - def find_method_run_step(self, method): - latest_run_step = RunStep.objects.filter( - pipelinestep__transformation_id=method.id).latest('run_id') - return latest_run_step - - def confirm(self, prompt): - print(prompt, end=' ') - confirmation = sys.stdin.readline().strip() - is_confirmed = confirmation.lower() == 'y' - return is_confirmed diff --git a/kive/container/management/commands/convert_pipelines.py b/kive/container/management/commands/convert_pipelines.py deleted file mode 100644 index dd8ecb98a..000000000 --- a/kive/container/management/commands/convert_pipelines.py +++ /dev/null @@ -1,435 +0,0 @@ -from __future__ import print_function -import logging -import os -import re -import shutil -from argparse import ArgumentDefaultsHelpFormatter -from tempfile import NamedTemporaryFile, mkdtemp -from zipfile import ZipFile, ZIP_DEFLATED - -from django.core.files.base import File -from django.core.management.base import BaseCommand -from django.conf import settings -from django.db import transaction -from six.moves import input - -from archive.models import Run -from constants import runstates -from container.models import Container, ContainerFamily, ContainerRun, Batch, ContainerArgument, ContainerLog -from pipeline.models import PipelineFamily, Pipeline - -logger = logging.getLogger(__name__) -CONVERTED_STATES = { - runstates.SUCCESSFUL_PK: ContainerRun.COMPLETE, - runstates.CANCELLED_PK: ContainerRun.CANCELLED, - runstates.FAILED_PK: ContainerRun.FAILED, - runstates.QUARANTINED_PK: ContainerRun.CANCELLED -} - - -def get_converting_pipeline_marker(container_id): - return 'Converting to container id {}.'.format(container_id) - - -def get_converted_pipeline_marker(container_id): - return 'Converted to container id {}.'.format(container_id) - - -def get_converted_family_marker(family_id): - return 'Converted to container family id {}.'.format(family_id) - - -def get_converted_run_marker(container_run_id): - return 'Converted to container run id {}.'.format(container_run_id) - - -def get_converted_batch_marker(container_batch_id): - return 'Converted to container batch id {}.'.format(container_batch_id) - - -def find_target_id(description, get_marker): - zero_marker = get_marker(0) - pattern = re.escape(zero_marker).replace('0', r'(\d+)') - match = re.search(pattern, description) - return match and match.group(1) - - -class Command(BaseCommand): - help = "Converts old pipelines into new archive containers." - - container_directory = os.path.normpath(os.path.join(settings.MEDIA_ROOT, Container.UPLOAD_DIR)) - - def add_arguments(self, parser): - parser.formatter_class = ArgumentDefaultsHelpFormatter - default_container_family = ContainerFamily.objects.filter(name='kive-default').first() - if default_container_family is None: - default_container_id = None - else: - default_container = default_container_family.containers.first() - default_container_id = default_container and default_container.id - parser.add_argument('--parent_container_id', - type=int, - default=default_container_id, - help='parent container to launch pipelines in') - parser.add_argument('--pipeline_id', - type=int, - help='pipeline to convert') - parser.add_argument('--batch_size', - type=int, - default=10, - help='number of runs to load in memory') - - def handle(self, - pipeline_id=None, - parent_container_id=None, - batch_size=None, - **options): - if parent_container_id is None: - raise ValueError('No parent container given.') - default_parent_container = Container.objects.get(id=parent_container_id) - - if pipeline_id is not None: - pipeline = Pipeline.objects.get(id=pipeline_id) - else: - pipeline = self.choose_pipeline() - if pipeline is None: - return - container_id = find_target_id(pipeline.revision_desc, - get_converted_pipeline_marker) - if container_id is not None: - raise ValueError('Pipeline id {} already converted.'.format( - pipeline.id)) - self.check_incomplete_runs(pipeline) - container_id = find_target_id(pipeline.revision_desc, - get_converting_pipeline_marker) - if container_id is None: - container = self.create_container(pipeline, default_parent_container) - else: - container = Container.objects.get(id=container_id) - - self.convert_runs(pipeline, container, batch_size) - converting_marker = get_converting_pipeline_marker(container.id) - converted_marker = get_converted_pipeline_marker(container.id) - pipeline.revision_desc = pipeline.revision_desc.replace( - converting_marker, - converted_marker) - pipeline.save() - - def convert_runs(self, pipeline, container, batch_size): - app = container.apps.get() - pipeline_runs = Run.objects.filter(pipeline=pipeline).order_by('id') - zero_marker = get_converted_run_marker(0) - sql_marker = zero_marker.split('0')[0] - unconverted_runs = pipeline_runs.exclude(description__icontains=sql_marker) - pipeline_run_count = pipeline_runs.count() - while True: - unconverted_run_count = unconverted_runs.count() - if not unconverted_run_count: - break - converted_run_count = pipeline_run_count - unconverted_run_count - print('Converted', converted_run_count, 'of', pipeline_run_count, 'runs.') - batch_runs = unconverted_runs[:batch_size] - for run in batch_runs: - with transaction.atomic(): - batch = run.runbatch - container_batch = self.find_or_create_batch(batch) - # noinspection PyProtectedMember - state = CONVERTED_STATES[run._runstate_id] - return_codes = [ - step.execrecord.generator.methodoutput.return_code - for step in run.runsteps.filter(execrecord__isnull=False)] - bad_return_codes = filter(None, return_codes) - return_code = bad_return_codes and bad_return_codes[-1] or 0 - container_run = ContainerRun.objects.create( - name=run.name, - description=run.description, - state=state, - app=app, - batch=container_batch, - user=run.user, - start_time=run.start_time, - end_time=run.end_time, - priority=run.priority, - return_code=return_code, - stopped_by=run.stopped_by) - container_run.copy_permissions(run) - - for run_input in run.inputs.all(): - argument = app.arguments.get(position=run_input.index, - type=ContainerArgument.INPUT) - container_run.datasets.create(argument=argument, - dataset=run_input.dataset) - for run_output in run.runoutputcables.all(): - if run_output.execrecord is None: - continue - output_index = run_output.pipelineoutputcable.output_idx - dataset = run_output.execrecord.execrecordouts.get().dataset - argument = app.arguments.get(position=output_index, - type=ContainerArgument.OUTPUT) - container_run.datasets.create(argument=argument, - dataset=dataset) - self.convert_logs(run, container_run) - container_run.set_md5() - container_run.submit_time = run.time_queued - container_run.save() - if run.description: - run.description += '\n' - run.description += get_converted_run_marker(container_run.id) - run.save() - print('Converted all {} runs to container id {}.'.format( - pipeline_run_count, - container.id)) - - def convert_logs(self, run, container_run): - work_path = mkdtemp(prefix='convert_pipelines_') - try: - stdout_path = os.path.join(work_path, 'stdout.txt') - stderr_path = os.path.join(work_path, 'stderr.txt') - for step in run.runsteps_in_order: - if step.execrecord is None: - continue - method_output = step.execrecord.generator.methodoutput - for log, log_path in ((method_output.output_log, - stdout_path), - (method_output.error_log, - stderr_path)): - try: - log_size = log.size - except ValueError: - # File was deleted. - log_size = 0 - if log_size: - header = '========== Step {} ==========\n'.format( - step.step_num) - self.copy_log(log, log_path, header) - for log_path, log_type in ((stdout_path, ContainerLog.STDOUT), - (stderr_path, ContainerLog.STDERR)): - with open(log_path, 'a'): - pass # Make sure the file exists. - container_run.load_log(log_path, log_type) - finally: - shutil.rmtree(work_path) - - def copy_log(self, log, dest_path, header): - log.open() - try: - with open(dest_path, 'a') as dest_log: - dest_log.write(header) - shutil.copyfileobj(log, dest_log) - finally: - log.close() - - def check_incomplete_runs(self, pipeline): - pipeline_runs = Run.objects.filter(pipeline=pipeline) - pipeline_run_count = pipeline_runs.count() - incomplete_runs = pipeline_runs.exclude( - _runstate__in=CONVERTED_STATES.keys()) - incomplete_run_count = incomplete_runs.count() - if incomplete_run_count: - raise ValueError( - '{} runs out of {} are incomplete. Cannot convert.'.format( - incomplete_run_count, - pipeline_run_count)) - - def find_or_create_batch(self, run_batch): - if run_batch is None: - return None - container_batch_id = find_target_id(run_batch.description, - get_converted_batch_marker) - if container_batch_id is not None: - return Batch.objects.get(container_batch_id) - with transaction.atomic(): - container_batch = Batch.objects.create( - name=run_batch.name, - description=run_batch.description, - user=run_batch.user) - container_batch.copy_permissions(run_batch) - if run_batch.description: - run_batch.description += '\n' - run_batch.description += get_converted_batch_marker( - container_batch.id) - return container_batch - - def create_container(self, pipeline, default_parent_container): - container = None - base_name = 'pipeline{}'.format(pipeline.id) - pipeline_config = self.build_pipeline_config(pipeline) - with NamedTemporaryFile(prefix=base_name, suffix='.zip') as f: - parent_containers = set() - copied_paths = set() - with ZipFile(f, 'w', ZIP_DEFLATED, allowZip64=True) as z: - for step in pipeline.steps.all(): - method = step.transformation.definite - if method.container is None and method.docker_image is not None: - raise ValueError('Convert docker image {}.'.format( - method.docker_image)) - parent_containers.add(method.container) - code_resource_revision = method.driver - install_path = code_resource_revision.coderesource.filename - self.add_script(code_resource_revision, - install_path, - copied_paths, - z) - for dependency in method.dependencies.all(): - code_resource_revision = dependency.requirement - install_path = os.path.join(dependency.path, - dependency.get_filename()) - self.add_script(code_resource_revision, - install_path, - copied_paths, - z) - parent_containers.discard(None) - if len(parent_containers) > 1: - raise ValueError('Found multiple containers: ' + - ', '.join(str(container) - for container in parent_containers)) - if not parent_containers: - parent_container = default_parent_container - else: - parent_container, = parent_containers - - with transaction.atomic(): - container_family = self.find_or_create_family(pipeline.family) - container = container_family.containers.create( - parent=parent_container, - tag=pipeline.revision_name, - description=pipeline.revision_desc, - user=pipeline.user, - file=File(f, name=base_name + '.zip'), - file_type=Container.ZIP) - container.copy_permissions(pipeline) - container.full_clean() - container.refresh_from_db() - container.write_archive_content(dict(pipeline=pipeline_config)) - container.created = pipeline.revision_DateTime - container.save() - - if pipeline.revision_desc: - pipeline.revision_desc += '\n' - pipeline_converting_marker = get_converting_pipeline_marker(container.id) - pipeline.revision_desc += pipeline_converting_marker - pipeline.save() - return container - - def build_pipeline_config(self, pipeline): - max_memory = 200 - max_threads = 1 - pipeline_config = dict(inputs=[], - steps=[], - outputs=[]) - for pipeline_input in pipeline.inputs.all(): - input_config = dict(dataset_name=pipeline_input.dataset_name, - x=pipeline_input.x, - y=pipeline_input.y) - pipeline_config['inputs'].append(input_config) - for step in pipeline.steps.all(): - method = step.transformation.definite - max_memory = max(max_memory, method.memory) - max_threads = max(max_threads, method.threads) - code_resource_revision = method.driver - install_path = code_resource_revision.coderesource.filename - inputs = [dict(dataset_name=cable.dest.dataset_name, - source_dataset_name=cable.source.definite.dataset_name, - source_step=cable.source_step) - for cable in step.cables_in.order_by('dest__dataset_idx')] - output_names = [o.dataset_name for o in method.outputs.all()] - dependencies = [] - for dependency in method.dependencies.all(): - dependencies.append(os.path.join(dependency.path, - dependency.get_filename())) - step_config = dict(inputs=inputs, - driver=install_path, - dependencies=dependencies, - outputs=output_names, - x=step.x, - y=step.y) - pipeline_config['steps'].append(step_config) - for pipeline_output in pipeline.outputs.all(): - cable = pipeline.outcables.get(output_idx=pipeline_output.dataset_idx) - output_config = dict( - dataset_name=pipeline_output.dataset_name, - source_dataset_name=cable.source.dataset_name, - source_step=cable.source_step, - x=pipeline_output.x, - y=pipeline_output.y) - pipeline_config['outputs'].append(output_config) - pipeline_config['default_config'] = dict(memory=max_memory, - threads=max_threads) - return pipeline_config - - def add_script(self, - code_resource_revision, - install_path, - copied_paths, - archive_file): - if install_path not in copied_paths: - driver_path = code_resource_revision.content_file.path - archive_file.write(driver_path, install_path) - copied_paths.add(install_path) - - def choose_pipeline(self): - # Look for pipeline in progress. - zero_marker = get_converting_pipeline_marker(0) - sql_search = zero_marker.split('0')[0] - pipelines = Pipeline.objects.filter(revision_desc__icontains=sql_search) - for pipeline in pipelines: - container_id = find_target_id(pipeline.revision_desc, - get_converting_pipeline_marker) - if container_id is not None: - print(pipeline) - # noinspection PyCompatibility - if input('In progress, continue? [Y]/N').upper() not in ('Y', ''): - return - return pipeline - - pipeline_families = PipelineFamily.objects.all() - family_map = {} - for i, pipeline_family in enumerate(pipeline_families, 1): - pipelines = pipeline_family.members.all() - unconverted_pipelines = [] - converted_pipelines = total_pipelines = 0 - for pipeline in pipelines: - container_id = find_target_id(pipeline.revision_desc, - get_converted_pipeline_marker) - if container_id is None: - unconverted_pipelines.append(pipeline) - else: - converted_pipelines += 1 - total_pipelines += 1 - family_map[pipeline_family.id] = unconverted_pipelines - print('{}: {} ({} of {} converted)'.format(i, - pipeline_family.name, - converted_pipelines, - total_pipelines)) - # noinspection PyCompatibility - choice = int(input('Pick a pipeline family: ')) - pipeline_family = pipeline_families[choice - 1] - unconverted_pipelines = family_map[pipeline_family.id] - for pipeline in unconverted_pipelines: - print('{}: {}'.format(pipeline.revision_number, - pipeline.revision_name)) - # noinspection PyCompatibility - choice = int(input('Pick a pipeline revision: ')) - pipeline = pipeline_family.members.get(revision_number=choice) - print(pipeline.revision_name) - return pipeline - - def find_or_create_family(self, pipeline_family): - container_family_id = find_target_id(pipeline_family.description, - get_converted_family_marker) - if container_family_id is not None: - return ContainerFamily.objects.get(id=container_family_id) - with transaction.atomic(): - container_family = ContainerFamily.objects.create( - name=pipeline_family.name, - description=pipeline_family.description, - user=pipeline_family.user) - container_family.copy_permissions(pipeline_family) - - if pipeline_family.description: - pipeline_family.description += '\n' - family_marker = get_converted_family_marker(container_family.id) - pipeline_family.description += family_marker - pipeline_family.save() - print('Created family id', container_family.id) - return container_family diff --git a/kive/container/management/commands/count_queries.py b/kive/container/management/commands/count_queries.py new file mode 100644 index 000000000..136506b46 --- /dev/null +++ b/kive/container/management/commands/count_queries.py @@ -0,0 +1,102 @@ +from __future__ import print_function +from collections import Counter +from datetime import datetime +import re +from operator import itemgetter + +from django.core.management import call_command +from django.core.management.base import BaseCommand +from django.core.urlresolvers import reverse, resolve +from django.db import connection +from rest_framework.test import APIRequestFactory, force_authenticate + +from metadata.models import kive_user + + +class Command(BaseCommand): + help = "Exercise the Run.is_complete() method for performance testing. " + + def handle(self, *args, **options): + self.count_queries(self.test_purge_synch) + + def test_purge_synch(self): + call_command('purge', synch=True) + + def count_queries(self, task): + """ Count the queries triggered by task, and print a summary. + + @param task: a callable that will trigger some database queries. + """ + start_count = len(connection.queries) + start_time = datetime.now() + result = task() + duration = datetime.now() - start_time + end_count = len(connection.queries) + print('{!r} after {} queries and {}.'.format( + result, + end_count - start_count, + duration)) + active_queries = connection.queries[start_count:] + min_time = duration.total_seconds() * 0.01 + slow_queries = [query + for query in active_queries + if float(query['time']) > min_time] + if slow_queries: + print('') + total_slow_time = sum(map(float, map(itemgetter('time'), slow_queries))) + total_time = sum(map(float, map(itemgetter('time'), active_queries))) + print("Slow queries ({:.2f}s for slow and {:.2f}s for all):".format( + total_slow_time, + total_time)) + max_display = 2000 + tail_size = 200 + for query in slow_queries: + display = str(query) + if len(display) > max_display: + display = (display[:max_display-tail_size] + + '...' + + display[-tail_size:]) + print(display) + + table_counts = Counter() + table_times = Counter() + for query in active_queries: + m = re.match('SELECT +"([^"]*)"', query['sql']) + if m: + table_counts[m.group(1)] += 1 + table_times[m.group(1)] += float(query['time']) + if m.group(1) == 'transformation_xputstructureXXX': + print(query['sql']) + print('') + print('Query counts:') + for table, count in table_counts.most_common(20): + print('{}: {}'.format(table, count)) + print('') + print('Query times:') + for table, time in table_times.most_common(20): + print('{}: {}'.format(table, time)) + + return result + + def test_ajax(self): + factory = APIRequestFactory() + path = '/api/datasets/' + view, _, _ = resolve(path) + request = factory.get( + path + '?is_granted=true&filters%5B0%5D%5Bkey%5D=uploaded' + '&filters%5B1%5D%5Bkey%5D=cdt&filters%5B1%5D%5Bval%5D=31' + '&page_size=8&page=1') + force_authenticate(request, user=kive_user()) + response = view(request).render() + data = response.render().data + return data['count'] + + def test_ajax_download(self): + factory = APIRequestFactory() + dataset_path = reverse('dataset-download', kwargs={'pk': 283134}) + view, _, _ = resolve(dataset_path) + request = factory.get(dataset_path) + force_authenticate(request, user=kive_user()) + response = view(request, pk=283134) + content = response.content + return content diff --git a/kive/datachecking/tests.py b/kive/datachecking/tests.py deleted file mode 100644 index 5c7d65134..000000000 --- a/kive/datachecking/tests.py +++ /dev/null @@ -1,156 +0,0 @@ -""" -This file demonstrates writing tests using the unittest module. These will pass -when you run "manage.py test". - -Replace this with more appropriate tests for your application. -""" - -from django.test import TestCase, skipIfDBFeature -from django.contrib.auth.models import User - -import shutil -import tempfile - -from constants import datatypes -import metadata.models -from datachecking.models import * -from librarian.models import * -import kive.testing_utils as tools - - -@skipIfDBFeature('is_mocked') -class BlankableTestCase(TestCase): - - def setUp(self): - self.user_doug = User.objects.create_user('doug', 'dford@deco.com', 'durrrrr') - self.user_doug.save() - self.user_doug.groups.add(metadata.models.everyone_group()) - self.user_doug.save() - - self.INT = metadata.models.Datatype.objects.get(pk=datatypes.INT_PK) - - self.canucks_lineup = """firstcol -22 -33 -17 - -23 -8 -""" - - -class BlankableColumn(BlankableTestCase): - - def setUp(self): - BlankableTestCase.setUp(self) - self.blankable_CDT = metadata.models.CompoundDatatype(user=self.user_doug) - self.blankable_CDT.save() - self.blankable_CDT.members.create(datatype=self.INT, column_name="firstcol", column_idx=1, - blankable=True) - self.blankable_CDT.clean() - - self.good_dataset = tools.make_dataset( - self.canucks_lineup, - self.blankable_CDT, - True, - self.user_doug, - "Dataset with blankable column", - "Canucks starting lineup", - None, - False - ) - - run_dir = tempfile.mkdtemp(prefix="dataset{}".format(self.good_dataset.pk)) - try: - self.good_dataset.dataset_file.open("r") - self.ccl = self.good_dataset.check_file_contents( - file_path_to_check=None, file_handle=self.good_dataset.dataset_file, - summary_path=run_dir, min_row=None, max_row=None, execlog=None, - checking_user=self.user_doug - ) - finally: - self.good_dataset.dataset_file.close() - shutil.rmtree(run_dir) - - def test_blank_on_blankable_column_OK(self): - """ - No error applied to a column that allows blanks. - """ - self.assertTrue(self.good_dataset.is_OK()) - - def test_clean_blank_on_blankable_column(self): - """ - There should be no BlankCell attached to an entry from a blankable column. - """ - ccl = self.good_dataset.content_checks.first() - baddata = BadData(contentchecklog=ccl) - baddata.save() - cell_error = baddata.cell_errors.create(row_num=4, column=self.blankable_CDT.members.first()) - - bc = BlankCell(cellerror = cell_error) - self.assertRaisesRegexp( - ValidationError, - 'Entry \(4,1\) of Dataset ".*" is blankable', - bc.clean - ) - - -class BlankCellNonBlankable(BlankableTestCase): - - def setUp(self): - BlankableTestCase.setUp(self) - self.test_CDT = metadata.models.CompoundDatatype(user=self.user_doug) - self.test_CDT.save() - self.test_CDT.members.create(datatype=self.INT, column_name="firstcol", column_idx=1) - self.test_CDT.clean() - - self.bad_dataset = tools.make_dataset( - self.canucks_lineup, - self.test_CDT, - True, - self.user_doug, - "Dataset with non-blankable column", - "Canucks starting lineup", - None, - False - ) - - run_dir = tempfile.mkdtemp(prefix="dataset{}".format(self.bad_dataset.pk)) - try: - self.bad_dataset.dataset_file.open("r") - self.ccl = self.bad_dataset.check_file_contents( - file_path_to_check=None, file_handle=self.bad_dataset.dataset_file, - summary_path=run_dir, min_row=None, max_row=None, - execlog=None, checking_user=self.user_doug - ) - finally: - self.bad_dataset.dataset_file.close() - shutil.rmtree(run_dir) - - def test_blank_on_non_blankable_column_creates_baddata(self): - """ - A blank cell causes an error when the CDT doesn't allow blanks. - """ - - self.assertFalse(self.bad_dataset.is_OK()) - - def test_blank_on_non_blankable_column_creates_cellerror(self): - """ - A blank cell creates a BlankCell object when the CDT doesn't allow blanks. - """ - self.assertEquals(self.bad_dataset.content_checks.count(), 1) - ccl = self.bad_dataset.content_checks.first() - self.assertTrue(ccl.is_fail()) - baddata = ccl.baddata - self.assertTrue(baddata.cell_errors.count(), 1) - cell_error = baddata.cell_errors.first() - self.assertEquals(cell_error.row_num, 4) - self.assertEquals(cell_error.column, self.test_CDT.members.first()) - self.assertIsNone(cell_error.constraint_failed) - - def test_blank_on_non_blankable_column_creates_blankcell(self): - """ - A blank cell creates a BlankCell object when the CDT doesn't allow blanks. - """ - cell_error = self.bad_dataset.content_checks.first().baddata.cell_errors.first() - self.assertTrue(cell_error.has_blank_error()) diff --git a/kive/fleet/management/__init__.py b/kive/fleet/management/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/kive/fleet/management/commands/__init__.py b/kive/fleet/management/commands/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/kive/fleet/management/commands/cable_helper.py b/kive/fleet/management/commands/cable_helper.py deleted file mode 100644 index 818cd1016..000000000 --- a/kive/fleet/management/commands/cable_helper.py +++ /dev/null @@ -1,29 +0,0 @@ -from django.core.management.base import BaseCommand - -from sandbox.execute import Sandbox -import file_access_utils - -import json -import logging -import time - -worker_logger = logging.getLogger("fleet.Worker") - - -class Command(BaseCommand): - help = 'Executes a cable based on the specified details.' - - def add_arguments(self, parser): - parser.add_argument( - "cable_execution_info_json", - help="JSON file containing cable execution information" - ) - - def handle(self, *args, **options): - worker_logger.debug("start time: %f" % time.time()) - file_access_utils.confirm_file_created(options["cable_execution_info_json"]) - with open(options["cable_execution_info_json"], "r") as f: - cable_execute_dict = json.loads(f.read()) - - Sandbox.finish_cable(cable_execute_dict) - worker_logger.debug("stop time: %f" % time.time()) diff --git a/kive/fleet/management/commands/fleetworker.py b/kive/fleet/management/commands/fleetworker.py deleted file mode 100644 index d869bdbbe..000000000 --- a/kive/fleet/management/commands/fleetworker.py +++ /dev/null @@ -1,14 +0,0 @@ -from django.core.management.base import BaseCommand -import fleet.workers - - -class Command(BaseCommand): - help = 'Worker process to execute pipelines.' - - def handle(self, *args, **options): - interface = fleet.workers.MPIWorkerInterface() - - worker = fleet.workers.Worker(interface) - worker.main_procedure() - - interface.close() diff --git a/kive/fleet/management/commands/runfleet.py b/kive/fleet/management/commands/runfleet.py deleted file mode 100644 index 33f070b8c..000000000 --- a/kive/fleet/management/commands/runfleet.py +++ /dev/null @@ -1,40 +0,0 @@ -from django.core.management.base import BaseCommand, CommandError -import fleet.workers - - -class Command(BaseCommand): - help = 'Launches the manager and worker_interfaces to execute pipelines.' - - def add_arguments(self, parser): - parser.add_argument( - "-q", - "--quit-idle", - dest="quit_idle", - action="store_true", - help="Shut down the fleet as soon as it is idle." - ) - parser.add_argument( - "-s", - "--stop-user", - dest="stop_user", - help="Username for the user that should stop all running tasks " - "at start up." - ) - parser.add_argument( - "--no-stop", - dest="no_stop", - action="store_true", - help="Do not stop running tasks at start up." - ) - - def handle(self, *args, **options): - try: - manager = fleet.workers.Manager(options["quit_idle"], - stop_username=options["stop_user"], - no_stop=options["no_stop"]) - except fleet.workers.ActiveRunsException as ex: - raise CommandError( - ('Found {} active runs. Use the --stop-user or ' - '--no-stop option.').format(ex.count)) - - manager.main_procedure() diff --git a/kive/fleet/management/commands/step_helper.py b/kive/fleet/management/commands/step_helper.py deleted file mode 100644 index c933bc0a8..000000000 --- a/kive/fleet/management/commands/step_helper.py +++ /dev/null @@ -1,50 +0,0 @@ -from django.core.management.base import BaseCommand - -from sandbox.execute import Sandbox -from fleet.exceptions import StopExecution -import file_access_utils - -import json -import sys -import time -import logging - -worker_logger = logging.getLogger("fleet.Worker") - - -class Command(BaseCommand): - help = 'Performs setup or bookkeeping for step execution.' - - def add_arguments(self, parser): - parser.add_argument( - "step_execution_info_json", - help="JSON file containing step execution information" - ) - - parser.add_argument( - "--bookkeeping", - action="store_true", - help="Whether to perform bookkeeping " - "(default is False, meaning this is setup rather than bookkeeping)" - ) - - def handle(self, *args, **options): - worker_logger.debug("start time: %f" % time.time()) - file_access_utils.confirm_file_created(options["step_execution_info_json"]) - with open(options["step_execution_info_json"], "r") as f: - step_execute_dict = json.loads(f.read()) - - if not options["bookkeeping"]: - try: - curr_run_step = Sandbox.step_execution_setup(step_execute_dict) - except StopExecution: - worker_logger.exception("Execution was stopped during setup.") - sys.exit(103) - - if curr_run_step.is_failed(): - sys.exit(101) - elif curr_run_step.is_cancelled(): - sys.exit(102) - else: - Sandbox.step_execution_bookkeeping(step_execute_dict) - worker_logger.debug("stop time: %f" % time.time()) diff --git a/kive/fleet/tests.py b/kive/fleet/tests.py deleted file mode 100644 index 2d7a990fe..000000000 --- a/kive/fleet/tests.py +++ /dev/null @@ -1,79 +0,0 @@ -from datetime import datetime -from mock import Mock -from unittest import TestCase - -from django.contrib.auth.models import User -from django_mock_queries.mocks import mocked_relations - -from archive.models import Run -from fleet.workers import Manager, ActiveRunsException - - -class ManagerMockTest(TestCase): - def setUp(self): - self.scheduler_class = Mock(name='MockSchedulerClass') - self.scheduler_class.slurm_is_alive.return_value = True - self.docker_class = Mock(name='MockDockerHandlerClass') - self.docker_class.docker_is_alive.return_value = True - - @mocked_relations(Run) - def test_simple(self): - Manager(slurm_sched_class=self.scheduler_class, - docker_handler_class=self.docker_class) - - @mocked_relations(Run) - def test_bad_slurm(self): - self.scheduler_class.slurm_is_alive.return_value = False - with self.assertRaisesRegexp(RuntimeError, - 'Slurm is down or badly configured.'): - Manager(slurm_sched_class=self.scheduler_class, - docker_handler_class=self.docker_class) - - @mocked_relations(Run) - def test_active_run_aborts(self): - Run.objects.create(start_time=datetime(2000, 12, 21)) - with self.assertRaises(ActiveRunsException) as result: - Manager(slurm_sched_class=self.scheduler_class, - docker_handler_class=self.docker_class) - - self.assertEqual(1, result.exception.count) - - @mocked_relations(Run) - def test_active_run_not_stopped(self): - Run.objects.create(start_time=datetime(2000, 12, 21)) - Manager(slurm_sched_class=self.scheduler_class, - no_stop=True, - docker_handler_class=self.docker_class) - - @mocked_relations(Run) - def test_completed_run_does_not_abort(self): - Run.objects.create(start_time=datetime(2000, 12, 21), - end_time=datetime(2000, 12, 22)) - Manager(slurm_sched_class=self.scheduler_class, - docker_handler_class=self.docker_class) - - @mocked_relations(Run, User) - def test_stopping_run_does_not_abort(self): - stop_user = User.objects.create(username='jblow') - Run.objects.create(start_time=datetime(2000, 12, 21), - stopped_by=stop_user) - Manager(slurm_sched_class=self.scheduler_class, - docker_handler_class=self.docker_class) - - @mocked_relations(Run, User) - def test_active_run_stopped(self): - stop_username = 'jblow' - User.objects.create(username=stop_username) - Run.objects.create(start_time=datetime(2000, 12, 21)) - Manager(slurm_sched_class=self.scheduler_class, - stop_username=stop_username, - docker_handler_class=self.docker_class) - - @mocked_relations(Run, User) - def test_unknown_user(self): - stop_username = 'jblow' - with self.assertRaises(User.DoesNotExist): - Manager(slurm_sched_class=self.scheduler_class, - stop_username=stop_username, - docker_handler_class=self.docker_class) - diff --git a/kive/fleet/tests_docker_build.py b/kive/fleet/tests_docker_build.py deleted file mode 100644 index 493150459..000000000 --- a/kive/fleet/tests_docker_build.py +++ /dev/null @@ -1,59 +0,0 @@ -from mock import patch -from subprocess import STDOUT, CalledProcessError -from unittest import TestCase -from six.moves.urllib.error import URLError -from fleet.docker_build import main - - -@patch('fleet.docker_build.check_call') -@patch('fleet.docker_build.check_output') -class DockerBuildTest(TestCase): - def test_simple(self, mock_check_output, mock_check_call): - args = ['my-image', 'http://my-host/my-project.git', 'v1.0'] - expected_inspect = ['docker', 'image', 'inspect', 'my-image:v1.0'] - mock_check_output.side_effect = CalledProcessError(1, expected_inspect) - expected_build = ['docker', - 'build', - '-t', 'my-image:v1.0', - 'http://my-host/my-project.git#tags/v1.0'] - - main(args) - - mock_check_output.assert_called_once_with(expected_inspect, - stderr=STDOUT) - mock_check_call.assert_called_once_with(expected_build) - - def test_file_protocol(self, mock_check_output, mock_check_call): - args = ['my-image', 'file:///home/alex/secret.txt', 'v1.0'] - - with self.assertRaisesRegexp(URLError, - 'Git repository must use http or https'): - main(args) - - mock_check_output.assert_not_called() - mock_check_call.assert_not_called() - - def test_fragment(self, mock_check_output, mock_check_call): - args = ['my-image', - 'http://my-host/my-project.git#heads/my-branch', - 'v1.0'] - - with self.assertRaisesRegexp(URLError, - 'Git repository may not contain #fragments'): - main(args) - - mock_check_output.assert_not_called() - mock_check_call.assert_not_called() - - def test_image_exists(self, mock_check_output, mock_check_call): - args = ['my-image', 'http://my-host/my-project.git', 'v1.0'] - expected_inspect = ['docker', 'image', 'inspect', 'my-image:v1.0'] - - with self.assertRaisesRegexp( - RuntimeError, - 'Docker image my-image:v1.0 already exists.'): - main(args) - - mock_check_output.assert_called_once_with(expected_inspect, - stderr=STDOUT) - self.assertEqual([], mock_check_call.call_args_list) diff --git a/kive/fleet/tests_dockerlib.py b/kive/fleet/tests_dockerlib.py deleted file mode 100755 index 659125d22..000000000 --- a/kive/fleet/tests_dockerlib.py +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/env python - -# some simple tests for the dockerlib module -import os -import shutil -from unittest import skipIf, skip - -import os.path as osp -import subprocess as sp - -import fleet.dockerlib as dockerlib -from django.conf import settings - -import tempfile - -from django.test import TestCase -import six - -TEST_DIR = osp.join(settings.KIVE_HOME, "fleet/dockerlib_test_files") - - -class DummyDockerLibTests(TestCase): - - def get_docker_handler_class(self): - return dockerlib.DummyDockerHandler - - def setUp(self): - self.addTypeEqualityFunc(str, self.assertMultiLineEqual) - self.docker_handler_class = self.get_docker_handler_class() - is_docker_alive = self.docker_handler_class.docker_is_alive() - self.assertTrue(is_docker_alive) - - def test_ident(self): - """Test the docker_ident call""" - idstr = self.docker_handler_class.docker_ident() - # print("Docker idents as:\n{}".format(idstr)) - assert isinstance(idstr, six.string_types) - - -@skipIf(not settings.RUN_DOCKER_TESTS, "Docker tests are disabled") -class DockerLibTests(DummyDockerLibTests): - - def get_docker_handler_class(self): - return dockerlib.DockerHandler - - @skip("This requires sudo permission. Enable test when we allow new images.") - def test_load_image01(self): - """Load a docker image from file""" - image_name = osp.join(TEST_DIR, "small-alpine.bz2") - exp_dct = {'REPOSITORY:TAG': 'alpine:latest', - 'CREATED AT': '2017-06-27 11:42:16 -0700 PDT', - 'IMAGE ID': '7328f6f8b418', - 'SIZE': '3.97MB', - 'DIGEST': ''} - img_info_dct = self.docker_handler_class._load_image_from_file(image_name) - # NOTE: we do not check the correctness of the CREATED_SINCE entry (the value - # will obviously depend on when the test is run). Instead, we simply delete it - # from the returned dict. - del img_info_dct[self.docker_handler_class.DOCKER_IMG_CREATED_SINCE] - assert exp_dct == img_info_dct, "unexpected info dict {}".format(img_info_dct) - - def test_load_image02(self): - """Loading a nonexistent docker image should raise an exception""" - with self.assertRaises(sp.CalledProcessError): - self.docker_handler_class._load_image_from_file("NONEXISTENT-IMAGE") - - def test_docker_images01(self): - """Sanity check the docker_images output.""" - out_lst = self.docker_handler_class.docker_images() - assert len(out_lst) > 0, "got zero docker images, cannot perform test" - expected_set = self.docker_handler_class.DOCKER_IMG_SET - for dct in out_lst: - assert expected_set == set(dct.keys()), "mangled image info dct keys {}".format(dct.keys()) - - def test_docker_images02(self): - """Search for a nonexistent docker image""" - out_lst = self.docker_handler_class.docker_images("BLAA") - assert len(out_lst) == 0, "got nonzero docker images!" - - def test_two_pipe_command(self): - expected_output = 'Beta\n' - - output = self.docker_handler_class._run_twopipe_command( - ['echo', 'Alpha\nBeta\nDelta'], - ['grep', 'B', '-']) - assert isinstance(output, six.string_types) - self.assertEqual(expected_output, output) - - def test_two_pipe_command_first_not_found(self): - expected_error2 = """\ -[Errno 2] No such file or directory: echoxxx Alpha -Beta -Delta | grep B -""" - expected_error3 = """\ -[Errno 2] No such file or directory: 'echoxxx': echoxxx Alpha -Beta -Delta | grep B -""" - - with self.assertRaises(OSError) as context: - self.docker_handler_class._run_twopipe_command( - ['echoxxx', 'Alpha\nBeta\nDelta'], - ['grep', 'B', '-']) - got_err = str(context.exception) - if got_err != expected_error2 and got_err != expected_error3: - raise RuntimeError("unexpected error message '{}' '{} '{}'".format(got_err, - expected_error2, - expected_error3)) - # self.assertEqual(expected_error, )) - - def test_two_pipe_command_second_not_found(self): - expected_error2 = """\ -[Errno 2] No such file or directory: echo Alpha -Beta -Delta | grepxxx B -""" - expected_error3 = """\ -[Errno 2] No such file or directory: 'grepxxx': echo Alpha -Beta -Delta | grepxxx B -""" - - with self.assertRaises(OSError) as context: - self.docker_handler_class._run_twopipe_command( - ['echo', 'Alpha\nBeta\nDelta'], - ['grepxxx', 'B', '-']) - got_err = str(context.exception) - if got_err != expected_error2 and got_err != expected_error3: - raise RuntimeError("unexpected error message '{}' '{} '{}'".format(got_err, - expected_error2, - expected_error3)) - # self.assertEqual(expected_error, ) - - def test_two_pipe_command_second_fails(self): - expected_error = ( - "Command '['grep', '-X', 'B', '-']' returned non-zero exit status 2") - expected_output = "" - - with self.assertRaises(sp.CalledProcessError) as context: - self.docker_handler_class._run_twopipe_command( - ['echo', 'Alpha\nBeta\nDelta'], - ['grep', '-X', 'B', '-']) - - self.assertEqual(expected_error, str(context.exception)) - self.assertEqual(expected_output, context.exception.output) - - def test_two_pipe_command_first_fails(self): - """ Failure in first process causes failure in second. """ - expected_error = ( - "Command '['grep', 'B', '-']' returned non-zero exit status 1") - expected_output = "" - - with self.assertRaises(sp.CalledProcessError) as context: - self.docker_handler_class._run_twopipe_command( - ['bzip2', 'unknown_file.bz2'], - ['grep', 'B', '-']) - - self.assertEqual(expected_error, str(context.exception)) - self.assertEqual(expected_output, context.exception.output) - - def test_gen_launch_cmd01(self): - """ Check sanity of generated a launch command """ - hoststepdir = tempfile.mkdtemp(prefix="kive_sandbox") - try: - input_file_paths = ["input1.dat", "input2.dat"] - output_file_paths = ["output1.dat", "output2.dat", "output3.dat"] - dep_paths = ["dep01.py", "dep02.py"] - image_id = "kive-default" - container_file = os.path.join(hoststepdir, "kive-default.simg") - with open(container_file, "w"): - pass # Touch the file. - for driver_name in [None, "my_driver_prog.py"]: - try: - retlst = self.docker_handler_class.generate_launch_args( - hoststepdir, - input_file_paths, - output_file_paths, - driver_name, - dep_paths, - image_id, - container_file) - assert isinstance(retlst, list), 'expected a list' - for s in retlst: - assert isinstance(s, six.string_types), 'expected a string' - lverb = False - if lverb: - print("got launch {}".format(retlst)) - except NotImplementedError: - pass - finally: - shutil.rmtree(hoststepdir) - -@skipIf(not settings.RUN_SINGULARITY_TESTS, "Singularity tests are disabled") -class SingularityDockerLibTests(DockerLibTests): - - def get_docker_handler_class(self): - return dockerlib.SingularityDockerHandler - - def placeholder_test_one(self): - # assert False, "force fail" - pass diff --git a/kive/fleet/tests_idletasks.py b/kive/fleet/tests_idletasks.py deleted file mode 100644 index e4f6bf00d..000000000 --- a/kive/fleet/tests_idletasks.py +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env python - -import errno -import inspect -import time -import os.path -import unittest -import shutil - -from django.conf import settings -from django.test import TestCase, skipIfDBFeature - -import fleet.slurmlib as slurmlib -from datetime import date, timedelta - -from fleet import dockerlib -from fleet.workers import Manager -from librarian.models import Dataset -from archive.models import MethodOutput - - -@skipIfDBFeature('is_mocked') -class IdleTaskTests(TestCase): - def setUp(self): - self.man = Manager(quit_idle=False, history=0, - slurm_sched_class=slurmlib.DummySlurmScheduler, - docker_handler_class=dockerlib.DummyDockerHandler) - - def tearDown(self): - self.man.slurm_sched_class.shutdown() - - def test_manager_ok(self): - """ Make sure we have a manager class.""" - self.assertIsNotNone(self.man) - - def test_add_idletask01(self): - """Adding a non-generator should raise and exception.""" - def test_func(myargs): - return myargs+1000.0 - - with self.assertRaises(RuntimeError): - self.man._add_idletask(test_func) - - def test_add_idletask02(self): - """Adding a generator should work.""" - def test_generator(myargs): - while True: - bla = (yield myargs) - bla += 1 - - gen = test_generator(100) - # just make sure our test is valid - self.assertTrue(inspect.isgenerator(gen), "The test is broken: test_gen is not a generator") - self.man._add_idletask(gen) - - def do_idle_tasks_test(self, lst, time_limit): - """Add three generators and call do_idle_tasks. - The generators modify lst if they are called.""" - def gen1(target): - while True: - (yield None) - target.append(1) - - def gen2(target): - while True: - (yield None) - target.append(2) - - def gen3(target): - while True: - (yield None) - target.append(3) - self.man._add_idletask(gen1(lst)) - self.man._add_idletask(gen2(lst)) - self.man._add_idletask(gen3(lst)) - self.man._do_idle_tasks(time_limit) - - def test_add_do_idle_tasks01(self): - """Add three generators. Calling do_idle_tasks with a big time_limit - should result in them being called all exactly once.""" - lst, time_limit = [], time.time() + 1000.0 - self.do_idle_tasks_test(lst, time_limit) - self.assertTrue(len(lst) == 3, "unexpected lst length") - self.assertTrue(set(lst) == {1, 2, 3}, "unexpected set") - - def test_add_do_idle_tasks02(self): - """Add three generators. Calling do_idle_tasks with a negative time_limit - should result in them being called all exactly never.""" - lst, time_limit = [], time.time() - 1000.0 - self.do_idle_tasks_test(lst, time_limit) - self.assertTrue(len(lst) == 0, "unexpected lst length") - self.assertTrue(set(lst) == set(), "unexpected set") - - def test_add_do_idle_tasks03(self): - """ Add four time-delayed generators. Waiting a specific time should - result in some of them being called and others not. - """ - def sleep_generator(target, task_id, secs_to_sleep): - while True: - (yield None) - target.append(task_id) - time.sleep(secs_to_sleep) - - wait_secs = 1.0 - lst = [] - self.man._add_idletask(sleep_generator(lst, 1, wait_secs)) - self.man._add_idletask(sleep_generator(lst, 2, wait_secs)) - self.man._add_idletask(sleep_generator(lst, 3, wait_secs)) - self.man._add_idletask(sleep_generator(lst, 4, wait_secs)) - time_limit = time.time() + 1.5*wait_secs - self.man._do_idle_tasks(time_limit) - self.assertTrue(len(lst) == 2, "unexpected lst length") - # NOTE: the order of the idle_tasks is not defined by the interface - # However, in fact the queue is rotated to the right... - self.assertTrue(set(lst) == {1, 4}, "unexpected set") - - def test_create_next_month_upload_dir01(self): - """ Test the creation of a monthly directory when the - Dataset dir is not present. - """ - dataset_dir = os.path.join(settings.MEDIA_ROOT, Dataset.UPLOAD_DIR) - date_str = (date.today() + timedelta(days=30)).strftime('%Y_%m') - next_dirname = os.path.join(dataset_dir, date_str) - # delete the dir iff it exists. - try: - shutil.rmtree(dataset_dir) - except os.error as e: - if e.errno != errno.ENOENT: - raise - gg = Dataset.idle_create_next_month_upload_dir() - self.man._add_idletask(gg) - time_limit = time.time() + 1000.0 - self.man._do_idle_tasks(time_limit) - self.assertTrue(os.path.exists(next_dirname), "directory was not made") - - def test_create_next_month_upload_dir02(self): - """ Test the creation of a monthly directory where Dataset may be present.""" - dataset_dir = os.path.join(settings.MEDIA_ROOT, Dataset.UPLOAD_DIR) - date_str = (date.today() + timedelta(days=30)).strftime('%Y_%m') - next_dirname = os.path.join(dataset_dir, date_str) - # delete the dir iff it exists. - try: - shutil.rmtree(next_dirname) - except os.error as e: - if e.errno != errno.ENOENT: - raise - gg = Dataset.idle_create_next_month_upload_dir() - self.man._add_idletask(gg) - time_limit = time.time() + 1000.0 - self.man._do_idle_tasks(time_limit) - self.assertTrue(os.path.exists(next_dirname), "directory was not made") - - def test_create_next_month_upload_dir03(self): - """ Test the creation of a monthly dir, where the dir is already present.""" - dataset_dir = os.path.join(settings.MEDIA_ROOT, Dataset.UPLOAD_DIR) - date_str = (date.today() + timedelta(days=30)).strftime('%Y_%m') - next_dirname = os.path.join(dataset_dir, date_str) - # make the directory iff it doesn't exist - if not os.path.exists(next_dirname): - os.makedirs(next_dirname) - gg = Dataset.idle_create_next_month_upload_dir() - self.man._add_idletask(gg) - time_limit = time.time() + 1000.0 - self.man._do_idle_tasks(time_limit) - self.assertTrue(os.path.exists(next_dirname), "directory was not made") - - def test_logfile_purge01(self): - # dataset_dir = os.path.join(settings.MEDIA_ROOT, Dataset.UPLOAD_DIR) - gg = MethodOutput.idle_logfile_purge() - self.man._add_idletask(gg) - for i in range(10): - # print "TEST", i - time_limit = time.time() + 10.0 - self.man._do_idle_tasks(time_limit) - - -if __name__ == "__main__": - unittest.main() diff --git a/kive/fleet/tests_slurmlib.py b/kive/fleet/tests_slurmlib.py deleted file mode 100644 index 3aaf0c4f3..000000000 --- a/kive/fleet/tests_slurmlib.py +++ /dev/null @@ -1,700 +0,0 @@ -#!/usr/bin/env python - -# some simple tests for the slurmlib module - -import os -import os.path as osp -import time -import subprocess as sp -import datetime - -# can't yet use pytest -- sniff -# import pytest -import unittest -from unittest import skipIf -import six -import fleet.slurmlib as slurmlib -from django.conf import settings - -from django.test import TestCase, skipIfDBFeature - -# NOTE: Here, select which SlurmScheduler to test. -# we select the DummySlurmScheduler by default, so that the automatic tests -# can run without slurm -# SlurmScheduler = slurmlib.SlurmScheduler -# SlurmScheduler = slurmlib.DummySlurmScheduler - - -ACC_STATE = slurmlib.BaseSlurmScheduler.ACC_STATE -ACC_JOB_ID = slurmlib.BaseSlurmScheduler.ACC_JOB_ID -ACC_PRIONUM = slurmlib.BaseSlurmScheduler.ACC_PRIONUM - -PRIO_LOW = slurmlib.BaseSlurmScheduler.MIN_PRIO -PRIO_MEDIUM = PRIO_LOW + 1 -PRIO_HIGH = PRIO_MEDIUM + 1 - -# this is the directory where the test jobs to submit to slurm will reside -# NOTE: the job number 02 must return a non-zero exit code for testing -FAIL_JOB_NUMBER = 2 -NUM_JOBS = 5 -TEST_DIR = osp.join(settings.KIVE_HOME, "fleet/slurm_test_files/slurmrundir") - - -def _submit_job_n(n, prio, afteroklst=None, afteranylst=None, sched_cls=None): - wdir = osp.join(TEST_DIR, "job%02d" % n) - jobname = "sleep%02d.sh" % n - return sched_cls.submit_job( - wdir, - jobname, - [], - prio, - 1, - osp.join(wdir, "out.txt"), - osp.join(wdir, "err.txt"), - after_okay=afteroklst, - after_any=afteranylst - ) - - -def submit_all(prio, sched_cls=None): - """ Submit all jobs with a certain priority.""" - return [_submit_job_n(i, prio, sched_cls=sched_cls) - for i in range(1, NUM_JOBS + 1)] - - -def get_accounting_info(jhandles=None, sched_cls=None): - curstates = sched_cls.get_accounting_info(job_handle_iter=jhandles) - if jhandles is not None and len(jhandles) > 0: - # check we have entries for all requested jhandles - jidset = set([jh.job_id for jh in jhandles]) - gotset = set(curstates.keys()) - assert gotset == jidset, "Did not get results from all submitted jobs" - - cls = sched_cls - for jid, dct in curstates.items(): - # makes sure all required fields are defined - assert cls.ACC_SET == set(dct.keys()), "inconsistent key set" - assert jid == dct[cls.ACC_JOB_ID] - prio = dct[cls.ACC_PRIONUM] - if prio is not None: - assert prio in cls.PRIO_SET, "invalid priority value" - for k in [cls.ACC_START_TIME, cls.ACC_END_TIME, cls.ACC_SUBMIT_TIME]: - tval = dct[k] - if tval is not None: - assert isinstance(tval, datetime.datetime), "wrong type of time field" - state = dct[cls.ACC_STATE] - assert state in cls.ALL_STATES, "illegal state" - return curstates - - -@skipIfDBFeature('is_mocked') # Doesn't use the database, but this test is slow. -class SlurmDummyTests(TestCase): - def setUp(self): - self.addTypeEqualityFunc(str, self.assertMultiLineEqual) - self.sched_cls = self.get_slurm_scheduler_class() - is_alive = self.sched_cls.slurm_is_alive(skip_extras=True) - if not is_alive: - raise RuntimeError("slurm is not alive") - - def get_slurm_scheduler_class(self): - return slurmlib.DummySlurmScheduler - - def tearDown(self): - self.sched_cls.shutdown() - - def test_callit01(self, lverb=False): - """ Should return 0 """ - n = 1 - if lverb: - print("---test_callit01", n) - wdir = osp.join(TEST_DIR, "job%02d" % n) - jobname = "sleep%02d.sh" % n - # arglst, stderr, stdout = [], None, None - arglst = [] - with open("/tmp/out.txt", "w") as stdout, open("/tmp/err.txt", "w") as stderr: - retval = slurmlib.callit(wdir, jobname, arglst, stdout, stderr) - if retval != 0: - print("the error is '%s'" % os.strerror(retval)) - assert retval == 0, "expected retval 0" - if lverb: - print("---END test_callit01", n) - - def test_callit02(self, lverb=False): - """ Should return 2 """ - n = FAIL_JOB_NUMBER - if lverb: - print("---test_callit01", n) - wdir = osp.join(TEST_DIR, "job%02d" % n) - jobname = "sleep%02d.sh" % n - arglst = [] - with open("/tmp/out.txt", "w") as stdout, open("/tmp/err.txt", "w") as stderr: - retval = slurmlib.callit(wdir, jobname, arglst, stdout, stderr) - assert retval == 2, "expected retval 2" - if lverb: - print("---END test_callit01", n) - - def test_is_alive(self): - """test_is_alive() should return True""" - is_alive = self.sched_cls.slurm_is_alive() - assert is_alive, "Calling is_alive fails" - - def test_slurm_ident(self): - """slurm_ident must return a string""" - idstr = self.sched_cls.slurm_ident() - assert isinstance(idstr, str), "slurm ident must be a string" - - def test_submit_job01(self, lverb=False): - """ Submitting this job should succeed.""" - if lverb: - print("--test_submit_job01") - jhandle = _submit_job_n(1, PRIO_MEDIUM, sched_cls=self.sched_cls) - if lverb: - print("submitted job", jhandle) - - def test_submit_job02(self, lverb=False): - """Submission should fail (nonexistent job script) - """ - if lverb: - print("--test_submit_job02") - prio = PRIO_MEDIUM - n, m = 1, 2 - wdir = osp.join(TEST_DIR, "job%02d" % n) - jobname = "sleep%02d.sh" % m - with self.assertRaises(sp.CalledProcessError): - self.sched_cls.submit_job( - wdir, - jobname, - [], - prio, - 1, - osp.join(wdir, "out.txt"), - osp.join(wdir, "err.txt"), - None - ) - - def test_submit_job03(self, lverb=False): - """Submission should fail (priority a string instead of int)""" - if lverb: - print("--test_submit_job03") - prio = 'illegal priostring' - n = 1 - wdir = osp.join(TEST_DIR, "job%02d" % n) - jobname = "sleep%02d.sh" % n - with self.assertRaises(RuntimeError): - self.sched_cls.submit_job( - wdir, - jobname, - [], - prio, - 1, - osp.join(wdir, "out.txt"), - osp.join(wdir, "err.txt"), - None - ) - - def test_submit_job04(self, lverb=False): - """Submit a job that should run, but returns a nonzero exit code. - I.e. submission should succeed, but the job should have a non-zero exit code. - """ - if lverb: - print("---test_submit_job04") - jhandle = _submit_job_n(FAIL_JOB_NUMBER, PRIO_MEDIUM, sched_cls=self.sched_cls) - if lverb: - print("successfully launched job %s, now waiting for its failure..." % jhandle) - time.sleep(2) - num_tries, i = 20, 0 - curstate = jhandle.get_state() - if lverb: - print("gotstate", curstate) - while (i < num_tries) and (curstate != self.sched_cls.FAILED): - if lverb: - print(i, "curstate...", curstate) - time.sleep(5) - curstate = jhandle.get_state() - i += 1 - assert curstate == self.sched_cls.FAILED, "failed to get a 'FAILED' state. got {}".format(curstate) - if lverb: - print("---test_submit_job04: Success, got an expected FAILED status") - - def test_submit_job07(self, lverb=False): - """Submission should fail (illegal cpu_number)""" - if lverb: - print("--test_submit_job07") - num_cpu = 0 - prio = PRIO_MEDIUM - n = 1 - wdir = osp.join(TEST_DIR, "job%02d" % n) - jobname = "sleep%02d.sh" % n - with self.assertRaises(sp.CalledProcessError): - self.sched_cls.submit_job( - wdir, - jobname, - [], - prio, - num_cpu, - osp.join(wdir, "out.txt"), - osp.join(wdir, "err.txt"), - None - ) - if lverb: - print("--test_submit_job07 SUCCESS") - - def test_dep_jobs01_okay(self, lverb=False): - """Submit one job dependent on the other with an after_okay dependency. - Both jobs should succeed.""" - if lverb: - print("--test_dep_jobs01_okay") - jobid_01 = _submit_job_n(1, PRIO_MEDIUM, sched_cls=self.sched_cls) - if lverb: - print("first job", jobid_01) - jobid_02 = _submit_job_n(3, PRIO_MEDIUM, [jobid_01], sched_cls=self.sched_cls) - if lverb: - print("dependent job", jobid_02) - my_handles = [jobid_01, jobid_02] - jobidlst = [j.job_id for j in my_handles] - time.sleep(2) - num_tries, i = 40, 0 - curstate = get_accounting_info(my_handles, sched_cls=self.sched_cls) - while i < num_tries and curstate[jobid_02.job_id][ACC_STATE] != self.sched_cls.COMPLETED: - if lverb: - print("step %02d:" % i, [curstate[jid][ACC_STATE] for jid in jobidlst]) - time.sleep(5) - curstate = get_accounting_info(my_handles, sched_cls=self.sched_cls) - i += 1 - if i == num_tries: - raise RuntimeError("test inconclusive: didn't wait long enough") - assert curstate[jobid_01.job_id][ACC_STATE] == self.sched_cls.COMPLETED, "job01: failed to run successfully" - assert curstate[jobid_02.job_id][ACC_STATE] == self.sched_cls.COMPLETED, "job02: failed to run successfully" - if lverb: - print("--test_dep_jobs01_okay SUCCESS") - - def test_dep_jobs01_any(self, lverb=False): - """Submit one job dependent on the other with an after_any dependency. - Both jobs should succeed.""" - if lverb: - print("--test_dep_jobs01_any") - jobid_01 = _submit_job_n(1, PRIO_MEDIUM, sched_cls=self.sched_cls) - if lverb: - print("first job", jobid_01) - jobid_02 = _submit_job_n(3, PRIO_MEDIUM, None, [jobid_01], sched_cls=self.sched_cls) - if lverb: - print("dependent job", jobid_02) - my_handles = [jobid_01, jobid_02] - jobidlst = [j.job_id for j in my_handles] - time.sleep(2) - num_tries, i = 40, 0 - curstate = get_accounting_info(my_handles, sched_cls=self.sched_cls) - while i < num_tries and curstate[jobid_02.job_id][ACC_STATE] != self.sched_cls.COMPLETED: - if lverb: - print("step %02d:" % i, [curstate[jid][ACC_STATE] for jid in jobidlst]) - time.sleep(5) - curstate = get_accounting_info(my_handles, sched_cls=self.sched_cls) - i += 1 - if i == num_tries: - raise RuntimeError("test inconclusive: didn't wait long enough") - if lverb: - print("FINAL STATE", [curstate[jid][ACC_STATE] for jid in jobidlst]) - assert curstate[jobid_01.job_id][ACC_STATE] == self.sched_cls.COMPLETED, "job01: failed to run successfully" - assert curstate[jobid_02.job_id][ACC_STATE] == self.sched_cls.COMPLETED, "job02: failed to run successfully" - if lverb: - print("--test_dep_jobs01_any SUCCESS") - - def test_dep_jobs02_ok(self, lverb=False): - """Submit job 01, and job 02 dependent on 01 with an after_ok dependency. - Job 01 will fail. Job 02 must be cancelled. - """ - if lverb: - print("--test_dep_jobs02_ok") - jobid_01 = _submit_job_n(FAIL_JOB_NUMBER, PRIO_MEDIUM, sched_cls=self.sched_cls) - if lverb: - print("first job that will fail:", jobid_01) - jobid_02 = _submit_job_n(3, PRIO_MEDIUM, [jobid_01], sched_cls=self.sched_cls) - if lverb: - print("dependent job:", jobid_02) - joblst = [jobid_01, jobid_02] - not_failed, num_tries, i = True, 40, 0 - curstate = None - while (i < num_tries) and not_failed: - time.sleep(2) - curstate = get_accounting_info(joblst, sched_cls=self.sched_cls) - if lverb: - print("step %02d:" % i, curstate[jobid_01.job_id][ACC_STATE], curstate[jobid_02.job_id][ACC_STATE]) - not_failed = curstate[jobid_01.job_id][ACC_STATE] != self.sched_cls.FAILED - i += 1 - if i == num_tries: - raise RuntimeError("test inconclusive: didn't wait long enough") - if lverb: - print("job01 state:", curstate[jobid_01.job_id]) - print("job02 state:", curstate[jobid_02.job_id]) - assert curstate[jobid_01.job_id][ACC_STATE] == self.sched_cls.FAILED, "unexpected state 01" - assert curstate[jobid_02.job_id][ACC_STATE] == self.sched_cls.CANCELLED, "unexpected state 02" - if lverb: - print("--test_dep_jobs02_ok SUCCESS") - - def test_dep_jobs02_any(self, lverb=False): - """Submit job 01, and job 02 dependent on 01 with an after_any dependency. - Job 01 will fail. Job 02 must run anyway. - """ - if lverb: - print("--test_dep_jobs02_any") - jobid_01 = _submit_job_n(FAIL_JOB_NUMBER, PRIO_MEDIUM, sched_cls=self.sched_cls) - if lverb: - print("first job that will fail:", jobid_01) - jobid_02 = _submit_job_n(3, PRIO_MEDIUM, None, [jobid_01], sched_cls=self.sched_cls) - if lverb: - print("dependent job:", jobid_02) - joblst = [jobid_01, jobid_02] - jidlst = [jh.job_id for jh in joblst] - if lverb: - print("waiting for job 01 to fail") - not_failed, num_tries, i = True, 40, 0 - curstate = None - while (i < num_tries) and not_failed: - time.sleep(2) - curstate = get_accounting_info(joblst, sched_cls=self.sched_cls) - if lverb: - print("step %02d:" % i, [curstate[jid][ACC_STATE] for jid in jidlst]) - not_failed = curstate[jobid_01.job_id][ACC_STATE] != self.sched_cls.FAILED - i += 1 - if i == num_tries: - raise RuntimeError("test inconclusive: didn't wait long enough") - # wait for jobid_02 to start running - if lverb: - print("OK, waiting for job 02 to run") - is_running, num_tries, i = curstate[jobid_02.job_id][ACC_STATE] == self.sched_cls.COMPLETED, 40, 0 - while (i < num_tries) and not is_running: - time.sleep(2) - curstate = get_accounting_info(joblst, sched_cls=self.sched_cls) - if lverb: - print("step %02d:" % i, [curstate[jid][ACC_STATE] for jid in jidlst]) - is_running = curstate[jobid_02.job_id][ACC_STATE] == self.sched_cls.COMPLETED - if i == num_tries: - raise RuntimeError("failed: job 02 did not complete") - if lverb: - print("job01 state:", curstate[jobid_01.job_id]) - print("job02 state:", curstate[jobid_02.job_id]) - state_02 = curstate[jobid_02.job_id][ACC_STATE] - ok_state_02 = state_02 == self.sched_cls.RUNNING or state_02 == self.sched_cls.COMPLETED - assert curstate[jobid_01.job_id][ACC_STATE] == self.sched_cls.FAILED, "unexpected state 01" - assert ok_state_02, "unexpected state 02" - if lverb: - print("--test_dep_jobs02_any SUCCESS") - - def test_dep_jobs01_multi(self, lverb=False): - """Submit job 01 that will fail. - Submit job o2 that will succeed. - Submit job 03, after_any on 01, and after_ok on 02. - Job 03 must be run. - """ - if lverb: - print("--test_dep_jobs01_multi") - jobid_01 = _submit_job_n(FAIL_JOB_NUMBER, PRIO_MEDIUM, sched_cls=self.sched_cls) - if lverb: - print("first job that will fail:", jobid_01) - jobid_02 = _submit_job_n(3, PRIO_MEDIUM, sched_cls=self.sched_cls) - if lverb: - print("second job that will succeed", jobid_02) - jobid_03 = _submit_job_n(1, PRIO_MEDIUM, [jobid_02], [jobid_01], sched_cls=self.sched_cls) - if lverb: - print("third job that should run", jobid_02) - joblst = [jobid_01, jobid_02, jobid_03] - jobidlst = [j.job_id for j in joblst] - still_running, num_tries, i = True, 40, 0 - curstate = None - while (i < num_tries) and still_running: - time.sleep(2) - curstate = get_accounting_info(joblst, sched_cls=self.sched_cls) - if lverb: - print("step %02d:" % i, [curstate[jid][ACC_STATE] for jid in jobidlst]) - still_running = (curstate[jobid_01.job_id][ACC_STATE] != self.sched_cls.FAILED) or\ - (curstate[jobid_02.job_id][ACC_STATE] != self.sched_cls.COMPLETED) - i += 1 - if i == num_tries: - raise RuntimeError("test inconclusive: didn't wait long enough") - state_01 = curstate[jobid_01.job_id][ACC_STATE] - state_02 = curstate[jobid_02.job_id][ACC_STATE] - state_03 = curstate[jobid_03.job_id][ACC_STATE] - if lverb: - print("state after loop:", [curstate[jid][ACC_STATE] for jid in jobidlst]) - assert state_01 == self.sched_cls.FAILED, "unexpected state 01: " + state_01 - assert state_02 == self.sched_cls.COMPLETED, "unexpected state 02: " + state_02 - assert state_03 in self.sched_cls.RUNNING_STATES, "unexpected state 03: " + state_03 - if lverb: - print("--test_dep_jobs01_multi SUCCESS") - - def test_cancel_jobs01(self, lverb=False): - """Submit a job, then cancel it""" - if lverb: - print("--test_cancel_jobs01") - jobid_01 = _submit_job_n(1, PRIO_MEDIUM, sched_cls=self.sched_cls) - if lverb: - print("submitted job", jobid_01) - print("wait for running status...") - num_tries, i, curstate = 40, 0, jobid_01.get_state() - while i < num_tries and curstate not in self.sched_cls.RUNNING_STATES: - if lverb: - print("step %02d:" % i, curstate) - time.sleep(2) - i += 1 - curstate = jobid_01.get_state() - assert curstate in self.sched_cls.RUNNING_STATES, "Job is not running, cannot test cancelling it" - if lverb: - print("job is running, now cancelling job 01...") - self.sched_cls.job_cancel(jobid_01) - if lverb: - print("wait for cancelled status....") - i, curstate = 0, jobid_01.get_state() - while i < num_tries and curstate in self.sched_cls.RUNNING_STATES: - if lverb: - print("step %02d:" % i, curstate) - time.sleep(5) - i += 1 - curstate = jobid_01.get_state() - assert curstate == self.sched_cls.CANCELLED, "job is not cancelled: got {}".format(curstate) - if lverb: - print("--test_cancel_jobs01 SUCCESS") - - def test_cancel_jobs02(self, lverb=False): - """Submit a job, then a second one dependent on the first. - When we cancel the first, slurm should cancel the second one as well. - """ - if lverb: - print("---test_cancel_jobs02") - jobid_01 = _submit_job_n(1, PRIO_MEDIUM, sched_cls=self.sched_cls) - if lverb: - print("started 01:", jobid_01) - time.sleep(2) - jobid_02 = _submit_job_n(3, PRIO_MEDIUM, [jobid_01], sched_cls=self.sched_cls) - if lverb: - print("started 02 (dependent on 01):", jobid_02) - joblst = [jobid_01, jobid_02] - jobidlst = [j.job_id for j in joblst] - are_ready, i, num_tries = False, 0, 40 - while i < num_tries and not are_ready: - time.sleep(2) - i += 1 - curstate = get_accounting_info(joblst, sched_cls=self.sched_cls) - if lverb: - print("step %02d:" % i, [curstate[jid][ACC_STATE] for jid in jobidlst]) - are_ready = curstate[jobid_01.job_id][ACC_STATE] in self.sched_cls.RUNNING_STATES - assert are_ready, "failed to submit the two jobs..." - if lverb: - print("OK, two jobs submitted, now cancelling job 01") - self.sched_cls.job_cancel(jobid_01) - check_tuple = (self.sched_cls.CANCELLED, self.sched_cls.CANCELLED) - curstate = get_accounting_info(joblst, sched_cls=self.sched_cls) - are_cancelled, i = False, 0 - while i < num_tries and not are_cancelled: - if lverb: - print("step %02d:" % i, [curstate[jid][ACC_STATE] for jid in jobidlst]) - time.sleep(2) - i += 1 - curstate = get_accounting_info(joblst, sched_cls=self.sched_cls) - are_cancelled = tuple([curstate[jid][ACC_STATE] for jid in jobidlst]) == check_tuple - if lverb: - print("final states", [curstate[jid][ACC_STATE] for jid in jobidlst]) - assert curstate[jobid_01.job_id][ACC_STATE] == self.sched_cls.CANCELLED,\ - "unexpected state 01: got {}".format(curstate[jobid_01.job_id][ACC_STATE]) - assert curstate[jobid_02.job_id][ACC_STATE] == self.sched_cls.CANCELLED,\ - "unexpected state 02: got {}".format(curstate[jobid_02.job_id][ACC_STATE]) - if lverb: - print("---test_cancel_jobs02 SUCCESS") - - def test_get_state_01(self, lverb=False): - """Submit a job, then follow its state using squeue. - If slurm accounting is not properly installed, we will never get - a COMPLETED result. - """ - if lverb: - print("--test_get_state_01") - jhandle = _submit_job_n(1, PRIO_MEDIUM, sched_cls=self.sched_cls) - if lverb: - print("submitted job", jhandle) - i, num_tries, has_finished = 0, 20, False - curstate = None - while (i < num_tries) and not has_finished: - curstate = jhandle.get_state() - if lverb: - print("step %02d:" % i, curstate) - has_finished = (curstate in self.sched_cls.STOPPED_SET) - time.sleep(5) - i += 1 - assert curstate == self.sched_cls.COMPLETED, "unexpected final state: got {}".format(curstate) - if lverb: - print("--test_get_state_01 SUCCESS") - - def test_set_priority_01(self, lverb=False): - """Start some jobs with a given priority, then change it. - See if this was successful. - """ - if lverb: - print("--test_set_priority_01") - # first, submit a number of high prio jobs in order to fill the queue - # we can only change the priority of a job if it is not yet running - low_prio = self.sched_cls.MIN_PRIO - high_prio = low_prio + 1 - for i in range(4): - submit_all(high_prio, sched_cls=self.sched_cls) - if lverb: - print("submitting low_prio jobs...") - jobhandles = submit_all(low_prio, sched_cls=self.sched_cls) - jobidlst = [jh.job_id for jh in jobhandles] - if lverb: - print("job_ids", jobidlst) - time.sleep(2) - cs = get_accounting_info(jobhandles, sched_cls=self.sched_cls) - priolst = [(cs[jid][ACC_STATE], cs[jid][ACC_PRIONUM]) for jid in jobidlst] - if lverb: - print("state+priority after submission", priolst) - any_done = any([state == self.sched_cls.COMPLETED for state, prio in priolst]) - if any_done: - raise RuntimeError("Test failed as jobs completed before we could change prio") - # now change the priority.. - self.sched_cls.set_job_priority(jobhandles, high_prio) - test_passed = False - while cs[jobidlst[0]][ACC_STATE] != self.sched_cls.PENDING and not test_passed: - if lverb: - print("waiting") - time.sleep(2) - cs = get_accounting_info(jobhandles, sched_cls=self.sched_cls) - priolst = [(cs[jid][ACC_STATE], cs[jid][ACC_PRIONUM]) for jid in jobidlst] - test_passed = all([prio == high_prio for state, prio in priolst]) - if lverb: - print(" after wait") - assert test_passed, "setting high prio failed" - if lverb: - print("Test passed") - if lverb: - cs = get_accounting_info(jobhandles, sched_cls=self.sched_cls) - priolst = [(cs[jid][ACC_STATE], cs[jid][ACC_PRIONUM]) for jid in jobidlst] - print("final states:", priolst) - - def test_set_priority_02(self): - """Set an illegal job priority type (str instead of int). - This should raise an exception.""" - low_prio = PRIO_LOW - jobhandles = submit_all(low_prio, sched_cls=self.sched_cls) - with self.assertRaises(RuntimeError): - self.sched_cls.set_job_priority(jobhandles, 'HI_PRIO') - - def test_set_priority_03(self): - """Set an job priority that is higher than MAX_PRIO. - This should simply set the actual priority to MAX_PRIO""" - low_prio = PRIO_LOW - jobhandles = submit_all(low_prio, sched_cls=self.sched_cls) - jidlst = [jh.job_id for jh in jobhandles] - time.sleep(1) - high_prio = PRIO_HIGH - self.sched_cls.set_job_priority(jobhandles, high_prio+10) - cs = get_accounting_info(jobhandles, sched_cls=self.sched_cls) - priolst = [cs[jid][ACC_PRIONUM] for jid in jidlst] - assert all([p == high_prio for p in priolst]), "Failed to set high priority" - - def test_acc_info_01(self, lverb=False): - """ Get_accounting_info must return information about all job handles - requested. - Where accounting info is not available, it must return the UNKNOWN state. - NOTE: in particular with slurm, accounting information is not available in the following - situation: - job A in PENDING in the queue - job B is dependent on A (with after_ok or after_any). - ==> there will be no information of job B by accounting. - """ - if lverb: - print("--test_acc_info_01:") - low_prio = PRIO_LOW - if lverb: - print("submitting low_prio jobs...") - jobhandles = submit_all(low_prio, sched_cls=self.sched_cls) - job01 = jobhandles[0] - job02 = _submit_job_n(1, PRIO_MEDIUM, [job01], sched_cls=self.sched_cls) - jobhandles.append(job02) - jidlst = [jh.job_id for jh in jobhandles] - time.sleep(1) - i, numtests, is_finished = 0, 40, False - while i < numtests and not is_finished: - cs = get_accounting_info(jobhandles, sched_cls=self.sched_cls) - if lverb: - print(i, [(cs[jid][ACC_STATE], cs[jid][ACC_PRIONUM]) for jid in jidlst]) - time.sleep(2) - is_finished = cs[job02.job_id][ACC_STATE] == self.sched_cls.COMPLETED - i += 1 - # -- - assert i < numtests,\ - "job02 failed to complete in 40 iterations. got state: {}".format(cs[job02.job_id][ACC_STATE]) - if lverb: - print("--test_acc_info_01(): SUCCESS") - - def test_multi_check_output_echo(self): - expected_output = u'Lorem ipsum\n' - - output = slurmlib.multi_check_output(['echo', 'Lorem', 'ipsum']) - assert isinstance(output, six.string_types), "string expected, but got {}".format(type(output)) - self.assertEqual(expected_output, output) - - def test_multi_check_output_echoxxx(self): - expected_py2_error = ( - '[Errno 2] No such file or directory: echoxxx Lorem ipsum') - expected_py3_error = ( - "[Errno 2] No such file or directory: 'echoxxx': echoxxx Lorem ipsum: 'echoxxx'") - with self.assertRaises(OSError) as context: - slurmlib.multi_check_output(['echoxxx', 'Lorem', 'ipsum']) - got_err = str(context.exception) - assert got_err == expected_py2_error or got_err == expected_py3_error, "unexpected error" - # self.assertEqual(expected_error, ) - - def show_squeue_jobs01(self, lverb=False): - """Submit all jobs with a low priority, then an additional one with high - priority. - List all jobs on the queue until the run queue is empty. - NOTE: this test will not terminate if some other process is adding jobs to the - queue. - NOTE: this routine does not assert anything or check for correctness. - It can be used for the user to see how priorities can/ should work. - Exactly how priorities are handled by slurm is a configuration issue, - and priorities could also be ignored. - """ - if lverb: - print("--test_squeue_jobs01") - low_prio = PRIO_LOW - hi_prio = PRIO_HIGH - if lverb: - print("submitting low_prio jobs...") - jh_lst = submit_all(low_prio, sched_cls=self.sched_cls) - time.sleep(1) - jobid_01 = _submit_job_n(1, hi_prio) - if lverb: - print("submitted a high prio job", jobid_01) - jh_lst.append(jobid_01) - is_done, i, num_tries = False, 0, 40 - job_state_dct = None - while (i < num_tries) and not is_done: - if lverb: - print("step %d/%d" % (i, num_tries)) - job_state_dct = get_accounting_info(jh_lst, sched_cls=self.sched_cls) - for j_state in sorted(job_state_dct.values(), key=lambda a: a[ACC_JOB_ID]): - # has_finished = (j_state["ST"] == 'CD' or j_state["ST"] == 'UKN') - if lverb: - print("%02d: %5s %s" % (i, j_state[ACC_JOB_ID], j_state[ACC_STATE])) - print() - is_done = job_state_dct[jobid_01.job_id][ACC_STATE] == self.sched_cls.COMPLETED - time.sleep(5) - i += 1 - if lverb: - print("exited loop...FINAL STATE:") - for j_state in sorted(job_state_dct.values(), key=lambda a: a[ACC_JOB_ID]): - print("FINAL: %5s %s" % (j_state[ACC_JOB_ID], j_state[ACC_STATE])) - print() - assert i < num_tries, "failed to wait for completed jobs!" - if lverb: - print("--test_squeue_jobs01 SUCCESS") - - -@skipIf(not settings.RUN_SLURM_TESTS, "Slurm tests disabled.") -class SlurmTests(SlurmDummyTests): - def get_slurm_scheduler_class(self): - return slurmlib.SlurmScheduler - - -if __name__ == "__main__": - unittest.main() diff --git a/kive/fleet/tests_slurmscheduler.py b/kive/fleet/tests_slurmscheduler.py deleted file mode 100644 index 0a71a0d1c..000000000 --- a/kive/fleet/tests_slurmscheduler.py +++ /dev/null @@ -1,276 +0,0 @@ - -import datetime -from unittest import skipIf - -from django.contrib.auth.models import User -from django.conf import settings -from django.test import skipIfDBFeature -from django.utils import timezone - -# import tempfile -# import os -# import shutil - -from fleet.workers import Manager -from kive.tests import BaseTestCases -import kive.testing_utils as tools -from metadata.models import everyone_group -from fleet.slurmlib import SlurmScheduler, DummySlurmScheduler -from fleet.dockerlib import DummyDockerHandler -from sandbox.tests_rm import BadRunTestsBase - - -def execute_simple_run(environment, slurm_sched_class): - """ - A helper function that creates a simple pipeline and executes a run. - - This also populates the object -- e.g. a TestCase or a FixtureBuilder -- - with some variables. - - Returns the Manager object that executed the run. - """ - tools.create_eric_martin_test_environment(environment) - tools.create_sandbox_testing_tools_environment(environment) - - user = User.objects.get(username='john') - # Everything in this pipeline will be a no-op, so all can be linked together - # without remorse. - p_basic = tools.make_first_pipeline("P_basic", "Innermost pipeline", user) - tools.create_linear_pipeline(p_basic, [environment.method_noop, environment.method_noop], "basic_in", "basic_out") - p_basic.family.grant_everyone_access() - p_basic.grant_everyone_access() - p_basic.create_outputs() - p_basic.save() - - # Set up a dataset with words in it called environment.dataset_words. - tools.make_words_dataset(environment) - - return Manager.execute_pipeline( - environment.user_bob, - p_basic, - [environment.dataset_words], - groups_allowed=[everyone_group()], - slurm_sched_class=slurm_sched_class, - docker_handler_class=DummyDockerHandler - ) - - -def execute_nested_run(environment, slurm_sched_class=DummySlurmScheduler): - """ - A helper function that creates a nested pipeline and executes a run. - - This also populates the object -- e.g. a TestCase or a FixtureBuilder -- - with some variables. - - Returns the Manager object that executed the run. - """ - tools.create_eric_martin_test_environment(environment) - tools.create_sandbox_testing_tools_environment(environment) - user = User.objects.get(username='john') - - # Everything in this pipeline will be a no-op, so all can be linked together - # without remorse. - p_basic = tools.make_first_pipeline("p_basic", "innermost pipeline", user) - tools.create_linear_pipeline(p_basic, [environment.method_noop, environment.method_noop], "basic_in", "basic_out") - p_basic.family.grant_everyone_access() - p_basic.grant_everyone_access() - p_basic.create_outputs() - p_basic.save() - - p_sub = tools.make_first_pipeline("p_sub", "second-level pipeline", user) - tools.create_linear_pipeline(p_sub, [p_basic, p_basic], "sub_in", "sub_out") - p_sub.family.grant_everyone_access() - p_sub.grant_everyone_access() - p_sub.create_outputs() - p_sub.save() - - p_top = tools.make_first_pipeline("p_top", "top-level pipeline", user) - tools.create_linear_pipeline(p_top, [p_sub, p_sub, p_sub], "top_in", "top_out") - p_top.family.grant_everyone_access() - p_top.grant_everyone_access() - p_top.create_outputs() - p_top.save() - - # Set up a dataset with words in it called environment.dataset_words. - tools.make_words_dataset(environment) - - return Manager.execute_pipeline( - environment.user_bob, - p_top, - [environment.dataset_words], - groups_allowed=[everyone_group()], - slurm_sched_class=slurm_sched_class - ) - - -@skipIf(not settings.RUN_SLURM_TESTS, "Slurm tests are disabled") -@skipIfDBFeature('is_mocked') -class SlurmExecutionTests(BaseTestCases.SlurmExecutionTestCase): - def test_simple_run(self): - """ - Execute a simple run. - """ - mgr = execute_simple_run(self, slurm_sched_class=SlurmScheduler) - run = mgr.get_last_run() - - self.check_run_OK(run) - - self.assertTrue(run.is_complete()) - self.assertTrue(run.is_successful()) - - self.assertIsNone(run.clean()) - self.assertIsNone(run.complete_clean()) - - def test_nested_run(self): - """ - Execute a nested run. - """ - mgr = execute_nested_run(self, slurm_sched_class=SlurmScheduler) - run = mgr.get_last_run() - - self.check_run_OK(run) - - self.assertTrue(run.is_complete()) - self.assertTrue(run.is_successful()) - - self.assertIsNone(run.clean()) - self.assertIsNone(run.complete_clean()) - - -# @skipIfDBFeature('is_mocked') -# class SlurmExecutionPathWithSpacesTests(SlurmExecutionTests): -# """ -# Repeat the same tests as SlurmExecutionTests, but with spaces in the Sandbox path. -# """ -# def setUp(self): -# self.media_root_original = settings.MEDIA_ROOT -# # Create this directory/probe that it exists. -# try: -# os.mkdir(settings.MEDIA_ROOT) -# except OSError: -# # It already exists. -# pass -# -# # Make a temporary directory whose name has spaces in it. -# self.base_with_spaces = tempfile.mkdtemp( -# suffix="Extra Folder With Spaces", -# dir=self.media_root_original -# ) -# # Just to be safe, we end MEDIA_ROOT with a directory named "Testing" as -# # this is consistent with the way we handle other tests that install fixture files. -# self.media_root_with_spaces = os.path.join(self.base_with_spaces, "Testing") -# settings.MEDIA_ROOT = self.media_root_with_spaces -# SlurmExecutionTests.setUp(self) -# -# def tearDown(self): -# SlurmExecutionTests.tearDown(self) -# settings.MEDIA_ROOT = self.media_root_original -# shutil.rmtree(self.base_with_spaces) - - -@skipIf(not settings.RUN_SLURM_TESTS, "Slurm tests are disabled") -@skipIfDBFeature('is_mocked') -class SlurmBadRunTests(BaseTestCases.SlurmExecutionTestCase, BadRunTestsBase): - """ - Tests a bad run using SlurmScheduler instead of DummySlurmScheduler. - - This inherits from the original test, so that we can test it both ways - (once in the original way, and once here). - """ - def setUp(self): - BaseTestCases.SlurmExecutionTestCase.setUp(self) - BadRunTestsBase.setUp(self) - - def tearDown(self): - BaseTestCases.SlurmExecutionTestCase.tearDown(self) - BadRunTestsBase.tearDown(self) - - def test_method_fails(self, - slurm_sched_class=SlurmScheduler, - docker_handler_class=DummyDockerHandler): - super(SlurmBadRunTests, self).test_method_fails(slurm_sched_class, - docker_handler_class) - - -class MockSlurmScheduler(DummySlurmScheduler): - """ A mocked -up slurm scheduler which will create NODE_FAIL events - with a job end time set to now() + my_time_delta. - The NODE_FAIL events are injected every second time get_accounting_info() is called, - in other times, the status information is passed through unchanged. - """ - count = 0 - my_time_delta = datetime.timedelta(seconds=-2*settings.NODE_FAIL_TIME_OUT_SECS) - name_tag = "PAST--" - - @classmethod - def slurm_ident(cls): - return "{}--{}--{}".format(cls.name_tag, - cls.my_time_delta, - super(MockSlurmScheduler, cls).slurm_ident()) - - @classmethod - def _mod_to_node_fail(cls, stat_dct): - """ Modify the status dict to a node_fail state - with an end time of now() + cls.my_time_delta - """ - now_time = datetime.datetime.now(timezone.get_current_timezone()) - end_time = now_time + cls.my_time_delta - for dct in stat_dct.values(): - dct[SlurmScheduler.ACC_STATE] = SlurmScheduler.NODE_FAIL - dct[SlurmScheduler.ACC_END_TIME] = end_time - - @classmethod - def get_accounting_info(cls, job_handle_iter=None): - # print("mock accounting {}".format(cls.count)) - stat_dct = super(MockSlurmScheduler, cls).get_accounting_info(job_handle_iter=job_handle_iter) - if cls.count % 2 == 0: - # print("overriding to NODE_FAIL: {}".format(cls.my_time_delta)) - cls._mod_to_node_fail(stat_dct) - else: - pass - # print("returning unchanged accounting info") - # print("RETURNING {}".format(stat_dct)) - cls.count += 1 - return stat_dct - - -class Recent_NF_Scheduler(MockSlurmScheduler): - count = 0 - my_time_delta = datetime.timedelta(seconds=0) - name_tag = "RECENT--" - - -@skipIfDBFeature('is_mocked') -class NodeFailExecutionTests(BaseTestCases.SlurmExecutionTestCase): - - def _sched_run_simple(self, slurm_sched_class): - # print("Running simple run with slurm : '%s'" % slurm_sched_class.slurm_ident()) - mgr = execute_simple_run(self, slurm_sched_class=slurm_sched_class) - return mgr - - def test_NF_future_run(self): - """ - Execute a simple run. - NODE_FAIL is set with a recent job end_time --> the job should complete - as normal. - """ - mgr = self._sched_run_simple(Recent_NF_Scheduler) - run = mgr.get_last_run() - - self.check_run_OK(run) - - self.assertTrue(run.is_complete()) - self.assertTrue(run.is_successful()) - - self.assertIsNone(run.clean()) - self.assertIsNone(run.complete_clean()) - - def test_NF_fail_run(self): - """ - Execute a simple run. - NODE_FAIL is set with a job end_time in the distant past --> - the job should fail. - """ - mgr = self._sched_run_simple(MockSlurmScheduler) - run = mgr.get_last_run() - self.assertTrue(run.is_failed()) diff --git a/kive/fleet/workers.py b/kive/fleet/workers.py deleted file mode 100644 index c974bfe24..000000000 --- a/kive/fleet/workers.py +++ /dev/null @@ -1,1206 +0,0 @@ -""" -Defines the manager and the "workers" that manage and carry out the execution of Pipelines. -""" - -from collections import deque -import logging -import time -import datetime -import itertools -import os -import glob -import shutil -import inspect -import six - -from django.conf import settings -from django.contrib.auth.models import User -from django.core.exceptions import ObjectDoesNotExist -from django.utils import timezone -from django.core.files import File -from django.db import transaction - -from archive.models import Dataset, Run, RunStep, RunSIC, MethodOutput, ExecLog -import file_access_utils -from sandbox.execute import Sandbox, sandbox_glob -from fleet.slurmlib import SlurmScheduler, DummySlurmScheduler, BaseSlurmScheduler -from fleet.dockerlib import DummyDockerHandler, DockerHandler, SingularityDockerHandler - - -mgr_logger = logging.getLogger("fleet.Manager") -foreman_logger = logging.getLogger("fleet.Foreman") - - -class ActiveRunsException(Exception): - def __init__(self, count): - super(ActiveRunsException, self).__init__( - 'Found {} active runs.'.format(count)) - self.count = count - - -class Manager(object): - """ - Coordinates the execution of pipelines. - - The manager is responsible for handling new Run requests and - creating Foreman objects to execute each one. - """ - def __init__( - self, - quit_idle=False, - history=0, - slurm_sched_class=SlurmScheduler, - docker_handler_class=DockerHandler, - stop_username=None, - no_stop=False, - singularity_handler_class=SingularityDockerHandler): - self.shutdown_exception = None - self.quit_idle = quit_idle - - # This keeps track of runs and the order in which they were most recently - # serviced. - self.runs = deque() - # This maps run -|-> foreman - self.runs_in_progress = {} - - # A queue of recently-completed Sandboxes, to a maximum specified by history. - self.history_queue = deque(maxlen=history) - - # A queue of functions to call during idle time. - self.idle_job_queue = deque() - - self.slurm_sched_class = slurm_sched_class - # when we start up, check to see whether slurm is running... - # we do want to be able to switch this off, e.g. when running tests. - slurm_is_ok = self.slurm_sched_class.slurm_is_alive() - mgr_logger.info("Slurm is OK: %s" % slurm_is_ok) - if not slurm_is_ok: - raise RuntimeError("Slurm is down or badly configured.") - # log some slurm information - mgr_logger.info("Slurm identifies as: '%s'" % self.slurm_sched_class.slurm_ident()) - - self.docker_handler_class = docker_handler_class - docker_is_ok = self.docker_handler_class.docker_is_alive() - mgr_logger.info("Docker is OK: %s" % docker_is_ok) - if not docker_is_ok: - raise RuntimeError("Docker is down or badly configured.") - mgr_logger.info("Docker identifies as: '%s'" % self.docker_handler_class.docker_ident()) - - self.singularity_handler_class = singularity_handler_class - singularity_is_ok = self.singularity_handler_class.docker_is_alive() - mgr_logger.info("Singularity is OK: %s" % singularity_is_ok) - if not singularity_is_ok: - raise RuntimeError("Singularity is down or badly configured.") - mgr_logger.info("Singularity identifies as: '%s'" % - self.singularity_handler_class.docker_ident()) - - if not no_stop: - if stop_username is None: - stop_user = None - else: - try: - stop_user = User.objects.get(username=stop_username) - except ObjectDoesNotExist: - raise User.DoesNotExist( - 'Username {!r} not found.'.format(stop_username)) - active_tasks = Run.objects.filter(start_time__isnull=False, - end_time__isnull=True, - stopped_by=None) - for task in active_tasks: - if stop_user is None: - raise ActiveRunsException(active_tasks.count()) - task.stopped_by = stop_user - task.save() - - def monitor_queue(self, time_to_stop): - """ - Monitor the queue for completed tasks. - - When a task is finished, it's handed off to the appropriate Foreman for handling. - """ - while True: # this will execute at least one time to help with starvation - try: - run = self.runs.popleft() - except IndexError: - # There are no active runs. - return - try: - foreman = self.runs_in_progress[run] - except KeyError: - raise RuntimeError("run in not found in runs_in_progress") - - foreman.monitor_queue() - run.refresh_from_db() - if run.is_complete(): - # All done, so remove it from our map. - self.runs_in_progress.pop(run) - if self.history_queue.maxlen > 0: - self.history_queue.append(foreman.sandbox) - else: - # Add it back to the end of the queue. - self.runs.append(run) - - if time.time() > time_to_stop: - break - - def _add_idletask(self, newidletask): - """Add a task for the manager to perform during her idle time. - The newidletask must be a generator that accepts a single argument of - type comparable to time.time() when it calls (yield) . - The argument provides the time limit after which a task must interrupt its task - and wait for the next allotted time. - - For example, the structure of an idle task would typically be: - - def my_idle_task(some_useful_args): - init_my_stuff() - while True: - time_to_stop = (yield) - if (time.time() < time_to_stop) and something_todo(): - do a small amount of work - """ - if inspect.isgenerator(newidletask): - # we prime the generator so that it advances to the first time that - # it encounters a 'time_to_stop = (yield)' statement. - next(newidletask) - self.idle_job_queue.append(newidletask) - else: - raise RuntimeError("add_idletask: Expecting a generator as a task") - - def _do_idle_tasks(self, time_limit): - """Perform the registered idle tasks once each, or until time runs out. - Idle tasks are called in a round-robin fashion. - """ - num_tasks = len(self.idle_job_queue) - num_done = 0 - while (time.time() < time_limit) and num_done < num_tasks: - mgr_logger.debug("Running an idle task....") - jobtodo = self.idle_job_queue[0] - jobtodo.send(time_limit) - self.idle_job_queue.rotate(1) - num_done += 1 - - def find_new_runs(self, time_to_stop): - """ - Look for and begin processing new runs. - """ - # Look for new jobs to run. We will also - # build in a delay here so we don't clog up the database. - mgr_logger.debug("Looking for new runs....") - pending_runs = Run.find_unstarted().order_by("time_queued").filter(parent_runstep__isnull=True) - mgr_logger.debug("Pending runs: {}".format(pending_runs)) - - for run_to_process in pending_runs: - if run_to_process.all_inputs_have_data(): - # lets try and run this run - foreman = Foreman(run_to_process, - self.slurm_sched_class, - self.docker_handler_class, - self.singularity_handler_class) - foreman.start_run() - - run_to_process.refresh_from_db() - if run_to_process.is_successful(): - # Well, that was easy. - mgr_logger.info("Run id %d, pipeline %s, user %s completely reused", - run_to_process.pk, - run_to_process.pipeline, - run_to_process.user) - if self.history_queue.maxlen > 0: - self.history_queue.append(foreman.sandbox) - else: - self.runs.append(run_to_process) - self.runs_in_progress[run_to_process] = foreman - mgr_logger.info("Started run id %d, pipeline %s, user %s", - run_to_process.pk, - run_to_process.pipeline, - run_to_process.user) - else: - # there is something wrong with the inputs (such as a maliciously moved input file) - mgr_logger.info("Missing input for run id %d, pipeline %s, user %s: RUN BEING CANCELLED", - run_to_process.pk, - run_to_process.pipeline, - run_to_process.user) - run_to_process.start(save=True) - run_to_process.cancel(save=True) - run_to_process.stop(save=True) - run_to_process.refresh_from_db() - mgr_logger.debug("Active runs: {}".format(self.runs_in_progress.keys())) - - if time.time() > time_to_stop: - # We stop, to avoid possible starvation if new tasks are continually added. - return - - def find_stopped_runs(self): - """ - Look for currently running Runs that have been stopped by a user. - """ - mgr_logger.debug("Looking for stopped runs....") - just_stopped_runs = Run.objects.filter(end_time__isnull=True, stopped_by__isnull=False) - - for run_to_stop in just_stopped_runs: - if run_to_stop not in self.runs_in_progress: - # This hasn't started yet, or is a remnant from a fleet crash/shutdown, - # so we can just skip this one. - mgr_logger.warn("Run (pk=%d) is not active. Cancelling steps/cables that were unfinished.", - run_to_stop.pk) - run_to_stop.cancel_components() - - with transaction.atomic(): - # Start anything that hadn't been started yet, i.e. in the Pending state. - if not run_to_stop.has_started(): - run_to_stop.start(save=True) - - # Cancel anything that is running (leave stuff that's Failing or already Cancelling). - if run_to_stop.is_running(): - run_to_stop.cancel(save=True) - - run_to_stop.stop(save=True) - continue - - self.runs_in_progress[run_to_stop].stop_run() # Foreman handles the stopping - - @staticmethod - def purge_sandboxes(): - # Next, look for finished jobs to clean up. - mgr_logger.debug("Checking for old sandboxes to clean up....") - - purge_interval = datetime.timedelta(days=settings.SANDBOX_PURGE_DAYS, - hours=settings.SANDBOX_PURGE_HOURS, - minutes=settings.SANDBOX_PURGE_MINUTES) - keep_recent = settings.SANDBOX_KEEP_RECENT - - purge_candidates = Run.objects.filter( - end_time__isnull=False, - end_time__lte=timezone.now()-purge_interval, - purged=False - ) - - # Retain the most recent ones for each PipelineFamily. - pfs_represented = purge_candidates.values_list("pipeline__family") - - ready_to_purge = [] - for pf in set(pfs_represented): - # Look for the oldest ones. - curr_candidates = purge_candidates.filter(pipeline__family=pf).order_by("end_time") - num_remaining = curr_candidates.count() - - ready_to_purge = itertools.chain( - ready_to_purge, - curr_candidates[:max(num_remaining - keep_recent, 0)] - ) - - for rtp in ready_to_purge: - mgr_logger.debug("Removing sandbox at %r.", rtp.sandbox_path) - try: - rtp.collect_garbage() - except OSError: - mgr_logger.error('Failed to purge sandbox at %r.', - rtp.sandbox_path, - exc_info=True) - rtp.purged = True # Don't try to purge it again. - rtp.save() - - # Next, look through the sandbox directory and see if there are any orphaned sandboxes - # to remove. - mgr_logger.debug("Checking for orphaned sandbox directories to clean up....") - - sdbx_path = os.path.join(settings.MEDIA_ROOT, settings.SANDBOX_PATH) - for putative_sdbx in glob.glob(os.path.join(sdbx_path, sandbox_glob)): - - # Remove this sandbox if there is no Run that is on record as having used it. - matching_rtps = Run.objects.filter( - sandbox_path__startswith=putative_sdbx, - ) - if not matching_rtps.exists(): - try: - path_to_rm = os.path.join(sdbx_path, putative_sdbx) - shutil.rmtree(path_to_rm) - except OSError as e: - mgr_logger.warning(e) - - def main_loop(self): - """ - Poll the database for new jobs, and handle running of sandboxes. - """ - # add any idle tasks that should be performed in the mainloop here - # -- - # purge old log files - self._add_idletask(MethodOutput.idle_logfile_purge()) - - time_to_purge = None - idle_counter = 0 - while True: - self.find_stopped_runs() - - poll_until = time.time() + settings.FLEET_POLLING_INTERVAL - self.find_new_runs(poll_until) - self.monitor_queue(poll_until) - - if time_to_purge is None or poll_until > time_to_purge: - self.purge_sandboxes() - time_to_purge = poll_until + settings.FLEET_PURGING_INTERVAL - - # Some jobs in the queue have been started: - # if we have time, do some idle tasks until poll_until and - # then check and see if anything has finished. - if settings.DO_IDLE_TASKS and time.time() < poll_until: - if idle_counter < settings.IDLE_TASK_FACTOR: - idle_counter += 1 - else: - self._do_idle_tasks(poll_until) - idle_counter = 0 - if self.quit_idle and not self.runs_in_progress: - mgr_logger.info('Fleet is idle, quitting.') - return - - try: - time.sleep(settings.SLEEP_SECONDS) - except KeyboardInterrupt: - return - - def main_procedure(self): - try: - self.main_loop() - mgr_logger.info("Manager shutting down.") - except Exception as ex: - mgr_logger.error("Manager failed.", exc_info=True) - self.shutdown_exception = ex - - for foreman in six.itervalues(self.runs_in_progress): - foreman.cancel_all_slurm_jobs() - self.slurm_sched_class.shutdown() - - @classmethod - def execute_pipeline(cls, - user, - pipeline, - inputs, - users_allowed=None, - groups_allowed=None, - name=None, - description=None, - slurm_sched_class=DummySlurmScheduler, - docker_handler_class=DummyDockerHandler, - singularity_handler_class=DummyDockerHandler): - """ - Execute the specified top-level Pipeline with the given inputs. - - This will create a run and start a fleet to run it. This is mainly used for testing, - and so a precondition is that sys.argv[1] is the management script used to invoke - the tests. - Also: the default slurm and docker handlers are set to the Dummy versions here. - """ - if settings.FLEET_POLLING_INTERVAL >= 1: - raise RuntimeError('FLEET_POLLING_INTERVAL has not been overridden.') - file_access_utils.create_sandbox_base_path() - - name = name or "" - description = description or "" - run = pipeline.pipeline_instances.create(user=user, name=name, description=description) - users_allowed = users_allowed or [] - groups_allowed = groups_allowed or [] - run.users_allowed.add(*users_allowed) - run.groups_allowed.add(*groups_allowed) - - for idx, curr_input in enumerate(inputs, start=1): - run.inputs.create(dataset=curr_input, index=idx) - - # Confirm that the inputs are OK. - pipeline.check_inputs(inputs) - - # The run is already in the queue, so we can just start the manager and let it exit - # when it finishes. - # noinspection PyTypeChecker - manager = cls(quit_idle=True, history=1, - slurm_sched_class=slurm_sched_class, - docker_handler_class=docker_handler_class, - singularity_handler_class=singularity_handler_class) - manager.main_procedure() - return manager - - def get_last_run(self): - """ - Retrieve the last completed run from the history. - - If no history is retained, return None. - """ - if self.shutdown_exception is not None: - raise self.shutdown_exception - if self.history_queue.maxlen == 0 or len(self.history_queue) == 0: - return None - - last_completed_sdbx = self.history_queue.pop() - return last_completed_sdbx.run - - -class Foreman(object): - """ - Coordinates the execution of a Run in a Sandbox. - """ - def __init__(self, - run, - slurm_sched_class, - docker_handler_class, - singularity_handler_class=None): - # tasks_in_progress tracks the Slurm IDs of currently running tasks: - # If the task is a RunStep: - # task -|--> { - # "setup": [setup handle], - # "driver": [driver handle], - # "bookkeeping": [bookkeeping handle], - # "info_path": [path of file specifying the details needed for execution] - # } - # If the task is a RunCable: - # task -|--> { - # "cable": [cable handle], - # "info_path": [path of file specifying details of execution] - # } - self.slurm_sched_class = slurm_sched_class - self.docker_handler_class = docker_handler_class - self.singularity_handler_class = singularity_handler_class - self.tasks_in_progress = {} - self.sandbox = Sandbox(run=run) - # A flag to indicate that this Foreman is in the process of terminating its Run and Sandbox. - self.shutting_down = False - self.priority = run.priority - - def is_node_fail(self, job_state): - return job_state == self.slurm_sched_class.NODE_FAIL - - def monitor_queue(self): - """ - Look to see if any of this Run's tasks are done. - """ - # For each task, get all relevant Slurm job handles. - our_slurm_jobs = [] - for task_dict in six.itervalues(self.tasks_in_progress): - if "cable" in task_dict: - our_slurm_jobs.append(task_dict["cable"]) - else: - if "bookkeeping" in task_dict: - our_slurm_jobs.append(task_dict["bookkeeping"]) - else: - our_slurm_jobs.append(task_dict["setup"]) - our_slurm_jobs.append(task_dict["driver"]) - task_accounting_info = self.slurm_sched_class.get_accounting_info(our_slurm_jobs) - - # These flags reflect the status from the actual execution, not - # the states of the RunComponents in the database. (We may have to - # use them to update said database states.) - node_fail_delta = datetime.timedelta(seconds=settings.NODE_FAIL_TIME_OUT_SECS) - terminated_during = "" - still_running = [] - - state_keyword = self.slurm_sched_class.ACC_STATE - raw_state_keyword = self.slurm_sched_class.ACC_RAW_STATE_STRING - start_keyword = self.slurm_sched_class.ACC_START_TIME - end_keyword = self.slurm_sched_class.ACC_END_TIME - return_code_keyword = self.slurm_sched_class.ACC_RETURN_CODE - - # self.tasks_in_progress may change during this loop, so we iterate over a list of the keys. - tasks = list(self.tasks_in_progress.keys()) - for task in tasks: - task_dict = self.tasks_in_progress[task] - raw_slurm_state = None - failed = cancelled = is_node_fail = False - # Check on the status of the jobs. - if isinstance(task, RunStep): - if "bookkeeping" not in task_dict: - setup_info = task_accounting_info.get(task_dict["setup"].job_id, None) - driver_info = task_accounting_info.get(task_dict["driver"].job_id, None) - - setup_state = None - if setup_info is not None and setup_info[state_keyword] != BaseSlurmScheduler.UNKNOWN: - setup_state = setup_info[state_keyword] - raw_setup_state = setup_info[raw_state_keyword] - - if setup_state is None or setup_state in self.slurm_sched_class.RUNNING_STATES: - # This is still going, so we move on. - still_running.extend([task_dict["setup"], task_dict["driver"]]) - continue - elif setup_state in self.slurm_sched_class.CANCELLED_STATES: - cancel_end_time = setup_info[end_keyword] - if self.is_node_fail(setup_state): - expiry_time = cancel_end_time + node_fail_delta - now_time = datetime.datetime.now(timezone.get_current_timezone()) - cancelled = expiry_time < now_time - foreman_logger.info("NODE_FAIL ({} vs {}): {} < {} ==> {}".format( - node_fail_delta, now_time-cancel_end_time, - expiry_time, now_time, cancelled)) - if not cancelled: - continue - else: - cancelled = True - terminated_during = "setup" - raw_slurm_state = raw_setup_state - elif setup_state in self.slurm_sched_class.FAILED_STATES: - # Something went wrong, so we get ready to bail. - failed = True - terminated_during = "setup" - raw_slurm_state = raw_setup_state - else: - assert setup_state in self.slurm_sched_class.SUCCESS_STATES, \ - "Unexpected Slurm state: {} (raw Slurm state: {})".format( - setup_state, - raw_setup_state - ) - # Having reached here, we know that setup is all clear, so check on the driver. - # Note that we don't check on whether it's in FAILED_STATES, because - # that will be handled in the bookkeeping stage. - driver_state = None - if (driver_info is not None and - driver_info[state_keyword] != BaseSlurmScheduler.UNKNOWN): - driver_state = driver_info[state_keyword] - raw_driver_state = driver_info[raw_state_keyword] - if driver_state is None or driver_state in self.slurm_sched_class.RUNNING_STATES: - still_running.append(task_dict["driver"]) - continue - elif driver_state in self.slurm_sched_class.CANCELLED_STATES: - # This was externally cancelled, so we get ready to bail. - cancel_end_time = driver_info[end_keyword] - if self.is_node_fail(driver_state): - expiry_time = cancel_end_time + node_fail_delta - now_time = datetime.datetime.now(timezone.get_current_timezone()) - cancelled = expiry_time < now_time - foreman_logger.info("NODE_FAIL ({} vs {}): {} < {} ==> {}".format( - node_fail_delta, now_time-cancel_end_time, - expiry_time, now_time, cancelled)) - if not cancelled: - continue - else: - cancelled = True - terminated_during = "driver" - raw_slurm_state = raw_driver_state - elif driver_info[start_keyword] is not None and driver_info[end_keyword] is not None: - # Having reached here, we know that the driver ran to completion, - # successfully or no, and has the start and end times properly set. - # As such, we remove the wrapped driver if necessary - # and fill in the ExecLog. - - # SCO do not remove for debugging.. - # if hasattr(task_dict["driver"], "wrapped_driver_path"): - # driver_path = task_dict["driver"].wrapped_driver_path - # if os.path.exists(driver_path): - # os.remove(task_dict["driver"].wrapped_driver_path) - - # Weirdly, task.log doesn't appear to be set even if you refresh task from the database, - # so we explicitly retrieve it. - # noinspection PyUnresolvedReferences - task_log = ExecLog.objects.get(record=task) - - with transaction.atomic(): - task_log.start_time = driver_info[start_keyword] - task_log.end_time = driver_info[end_keyword] - task_log.methodoutput.return_code = driver_info[return_code_keyword] - - step_execute_info = self.sandbox.step_execute_info[(task.parent_run, task.pipelinestep)] - # Find the stdout and stderr log files from their prefixes - # (since the full filename is produced using some Slurm macros). - stdout_pattern = os.path.join( - step_execute_info.log_dir, - "{}*.txt".format(step_execute_info.driver_stdout_path_prefix())) - stdout_log = glob.glob(stdout_pattern) - expected = "Expected 1 stdout log in {}, found {}".format( - stdout_pattern, - stdout_log) - assert len(stdout_log) == 1, expected - - stderr_pattern = os.path.join( - step_execute_info.log_dir, - "{}*.txt".format(step_execute_info.driver_stderr_path_prefix())) - stderr_log = glob.glob(stderr_pattern) - expected = "Expected 1 stderr log in {}, found {}".format( - stderr_pattern, - stderr_log) - assert len(stderr_log) == 1, expected - - with open(stdout_log[0], "rb") as f: - task_log.methodoutput.output_log.save( - os.path.basename(f.name), - File(f)) - with open(stderr_log[0], "rb") as f: - task_log.methodoutput.error_log.save( - os.path.basename(f.name), - File(f)) - - task_log.methodoutput.save() - task_log.save() - - # Now, we can submit the bookkeeping task, and the next time around we'll - # watch for it. - bookkeeping_job = self.submit_runstep_bookkeeping(task, task_dict["info_path"]) - self.tasks_in_progress[task]["bookkeeping"] = bookkeeping_job - continue - - else: - assert (driver_state in self.slurm_sched_class.FAILED_STATES - or driver_state in self.slurm_sched_class.SUCCESS_STATES), \ - "Unexpected Slurm state: {} (raw Slurm state: {})".format( - driver_state, - raw_driver_state - ) - - # The driver is finished, but sacct hasn't properly gotten the start and end times. - # For all intents and purposes, this is still running. - foreman_logger.debug( - "Driver of task %s appears complete but sacct hasn't set start/end times properly", - str(task) - ) - foreman_logger.debug( - "sacct returned the following: %s", - str(driver_info) - ) - still_running.append(task_dict["driver"]) - continue - - else: - bookkeeping_info = task_accounting_info.get(task_dict["bookkeeping"].job_id, None) - # Check on the bookkeeping script. - bookkeeping_state = None - if (bookkeeping_info is not None and - bookkeeping_info[state_keyword] != BaseSlurmScheduler.UNKNOWN): - bookkeeping_state = bookkeeping_info[state_keyword] - raw_bookkeeping_state = bookkeeping_info[raw_state_keyword] - if bookkeeping_state is None or bookkeeping_state in self.slurm_sched_class.RUNNING_STATES: - still_running.append(task_dict["bookkeeping"]) - continue - elif bookkeeping_state in self.slurm_sched_class.CANCELLED_STATES: - cancel_end_time = bookkeeping_info[end_keyword] - if self.is_node_fail(bookkeeping_state): - expiry_time = cancel_end_time + node_fail_delta - now_time = datetime.datetime.now(timezone.get_current_timezone()) - cancelled = expiry_time < now_time - foreman_logger.info("NODE_FAIL ({} vs {}): {} < {} ==> {}".format( - node_fail_delta, now_time-cancel_end_time, - expiry_time, now_time, cancelled)) - if not cancelled: - continue - else: - cancelled = True - terminated_during = "bookkeeping" - raw_slurm_state = raw_bookkeeping_state - elif bookkeeping_state in self.slurm_sched_class.FAILED_STATES: - # Something went wrong, so we bail. - failed = True - terminated_during = "bookkeeping" - raw_slurm_state = raw_bookkeeping_state - else: - assert bookkeeping_state in self.slurm_sched_class.SUCCESS_STATES, \ - "Unexpected Slurm state: {} (raw Slurm state: {})".format( - bookkeeping_state, - raw_bookkeeping_state - ) - - else: - cable_info = task_accounting_info.get(task_dict["cable"].job_id, None) - cable_state = None - if (cable_info is not None and - cable_info[state_keyword] != BaseSlurmScheduler.UNKNOWN): - cable_state = cable_info[state_keyword] - raw_cable_state = cable_info[raw_state_keyword] - - if cable_state is None or cable_state in self.slurm_sched_class.RUNNING_STATES: - # This is still going, so we move on. - still_running.append(task_dict["cable"]) - continue - elif cable_state in self.slurm_sched_class.CANCELLED_STATES: - cancel_end_time = cable_info[end_keyword] - if self.is_node_fail(cable_state): - expiry_time = cancel_end_time + node_fail_delta - now_time = datetime.datetime.now(timezone.get_current_timezone()) - cancelled = expiry_time < now_time - foreman_logger.info("NODE_FAIL ({} vs {}): {} < {} ==> {}".format( - node_fail_delta, now_time-cancel_end_time, - expiry_time, now_time, cancelled)) - if not cancelled: - continue - else: - cancelled = True - terminated_during = "cable processing" - raw_slurm_state = raw_cable_state - elif cable_state in self.slurm_sched_class.FAILED_STATES: - # Something went wrong, so we get ready to bail. - failed = True - terminated_during = "cable processing" - raw_slurm_state = raw_cable_state - else: - assert cable_state in self.slurm_sched_class.SUCCESS_STATES, \ - "Unexpected Slurm state: {} (raw Slurm state: {})".format( - cable_state, - raw_cable_state - ) - - # Having reached here, we know we're done with this task. - if is_node_fail: - foreman_logger.debug('Run "%s" (pk=%d, Pipeline: %s, User: %s) NODE_FAIL while ' - 'handling task %s (pk=%d) during %s (raw Slurm state: %s)', - self.sandbox.run, - self.sandbox.run.pk, - self.sandbox.pipeline, - self.sandbox.user, - task, - task.pk, - terminated_during, - raw_slurm_state) - if failed or cancelled: - foreman_logger.error( - 'Run "%s" (Pipeline: %s, User: %s) %s while handling task %s (pk=%d) during %s ' - '(raw Slurm state: %s)', - self.sandbox.run, - self.sandbox.pipeline, - self.sandbox.user, - "failed" if failed else "cancelled", - task, - task.pk, - terminated_during, - raw_slurm_state - ) - - task.refresh_from_db() - if not task.is_complete(): # it may have been properly handled already in setup or bookkeeping - if failed: - assert task.has_started() - # Mark it as failed. - task.finish_failure() - else: - task.cancel() - - # At this point, the task has either run to completion or been - # terminated either through failure or cancellation. - # The worker_finished routine will handle it from here. - self.tasks_in_progress.pop(task) - self.worker_finished(task) - - # Lastly, check if the priority has changed. - self.sandbox.run.refresh_from_db() - priority_changed = self.priority != self.sandbox.run.priority - self.priority = self.sandbox.run.priority - if priority_changed: - foreman_logger.debug( - 'Changing priority of Run "%s" (Pipeline: %s, User: %s) to %d', - self.sandbox.run, - self.sandbox.pipeline, - self.sandbox.user, - self.priority - ) - self.slurm_sched_class.set_job_priority(still_running, self.priority) - - def start_run(self): - """ - Receive a request to start a pipeline running. - This is the entry point for the foreman after having been created - by the Manager. - """ - self.sandbox.advance_pipeline() - - # Refresh run_to_start. - self.sandbox.run.refresh_from_db() - - # If we were able to reuse throughout, then we're totally done. Otherwise we - # need to do some bookkeeping. - if self.sandbox.run.is_successful(): - foreman_logger.info( - 'Run "%s" (Pipeline: %s, User: %s) completely reused successfully', - self.sandbox.run, - self.sandbox.run.pipeline, - self.sandbox.run.user - ) - - elif self.sandbox.run.is_failing() or self.sandbox.run.is_cancelling(): - # The run failed somewhere in preparation. This hasn't affected any of our maps yet, so we - # just report it and discard it. - status_str = "failed" if self.sandbox.run.is_failing() else "cancelled" - foreman_logger.info( - 'Run "%s" (Pipeline: %s, User: %s) %s before execution', - self.sandbox.run, - self.sandbox.run.pipeline, - self.sandbox.run.user, - status_str - ) - self.sandbox.run.cancel_components() - self.sandbox.run.stop(save=True) - - else: - self.submit_ready_tasks() - - return self.sandbox - - def mop_up(self): - """ - Mop up the sandbox after cancellation or failure. - """ - # Mark this sandbox as in the process of shutting down. - self.shutting_down = True - - # Cancel all parts of the run that aren't currently processing. - steps_processing = [] - incables_processing = [] - outcables_processing = [] - for task in self.tasks_in_progress: - if isinstance(task, RunStep): - steps_processing.append(task) - elif isinstance(task, RunSIC): - incables_processing.append(task) - else: - outcables_processing.append(task) - self.sandbox.run.cancel_components( - except_steps=steps_processing, - except_incables=incables_processing, - except_outcables=outcables_processing - ) - - def submit_ready_tasks(self): - """ - Go through the task queue, submitting all ready tasks to Slurm. - """ - for task in self.sandbox.hand_tasks_to_fleet(): - if isinstance(task, RunStep): - slurm_info = self.submit_runstep(task) - else: - slurm_info = self.submit_runcable(task) - - self.tasks_in_progress[task] = slurm_info - - def submit_runcable(self, runcable): - """ - Submit a RunCable for processing. - - Return a dictionary containing the SlurmJobHandle for the cable helper, - as well as the path of the execution info dictionary file used by the step. - """ - cable_slurm_handle, cable_execute_dict_path = self.slurm_sched_class.submit_runcable( - runcable, - self.sandbox - ) - return { - "cable": cable_slurm_handle, - "info_path": cable_execute_dict_path - } - - def submit_runstep(self, runstep): - """ - Submit a RunStep to Slurm for processing. - - A RunStep proceeds in four parts: - - setup - - driver - - Foreman finishes the ExecLog - - bookkeeping - The first two parts, and the last part, are handled by tasks submitted - to Slurm. The third part is performed by the Foreman. This function - submits the first two, with a dependency so that - the driver relies on the setup. - - Return a dictionary containing SlurmJobHandles for each, as well as - the path of the execution info dictionary file used by the step. - """ - # First, serialize the task execution information. - step_info = self.sandbox.step_execute_info[(runstep.run, runstep.pipelinestep)] - - setup_slurm_handle, step_execute_dict_path = self.slurm_sched_class.submit_step_setup( - runstep, - self.sandbox - ) - - driver_slurm_handle = self.sandbox.submit_step_execution( - step_info, - after_okay=[setup_slurm_handle], - slurm_sched_class=self.slurm_sched_class, - docker_handler_class=self.docker_handler_class, - singularity_handler_class=self.singularity_handler_class - ) - - return { - "setup": setup_slurm_handle, - "driver": driver_slurm_handle, - "info_path": step_execute_dict_path - } - - def submit_runstep_bookkeeping(self, runstep, info_path): - """ - Submit the bookkeeping part of a RunStep. - - This is to be called after the driver part of a RunStep has been finished - and the Foreman has completed the ExecLog. It uses the same step execution - information path as submit_runstep. - """ - return self.slurm_sched_class.submit_step_bookkeeping( - runstep, - info_path, - self.sandbox - ) - - def worker_finished(self, finished_task): - """Handle bookkeeping when a worker finishes.""" - foreman_logger.info( - "Run %s reports task with PK %d is finished", - self.sandbox.run, - finished_task.pk - ) - - # Mark this task as having finished. - task_execute_info = self.sandbox.get_task_info(finished_task) - - # Is anything from the run actively processing? - tasks_currently_running = len(self.tasks_in_progress) > 0 - # Recall: - # a RunStep gives you the step coordinates - # a RunSIC gives you its parent step coordinates - # a RunOutputCable gives you the parent run coordinates - # finished_task_coords[idx] is the component coordinate in - # the subrun idx levels deep (0 means top-level run). - finished_task_coords = finished_task.get_coordinates() - if finished_task.is_outcable(): - # Add a dummy entry at the end so that the 0th to 2nd-last coordinates - # give the sub-run coordinates in all cases. - finished_task_coords = finished_task_coords + (None,) - - # At position i, this list denotes whether any other tasks from the sub-run - # i levels deep (0 means top level run) are currently running. - subrun_tasks_currently_running = [False for _ in finished_task_coords] - subrun_tasks_currently_running[0] = tasks_currently_running - - # for task_info in self.tasks_in_progress.itervalues(): - for running_task in self.tasks_in_progress: - running_task_coords = running_task.get_coordinates() - if running_task.is_outcable(): - running_task_coords = running_task_coords + (None,) - - # If either finished_task_coords and running_task_coords are length 1 (i.e. one - # directly belongs to the top-level run), this does nothing. - for coord in range(1, min(len(finished_task_coords), len(running_task_coords))): - if finished_task_coords[coord] == running_task_coords[coord]: - subrun_tasks_currently_running[coord] = True - else: - # Nothing nested deeper can belong to the same sub-run. - break - - # If this run has failed (either due to this task or another), - # we mop up. - clean_up_now = False - self.sandbox.run.refresh_from_db() - stop_subruns_if_possible = False - - finished_task.refresh_from_db() - if finished_task.is_successful(): - if self.sandbox.run.is_failing() or self.sandbox.run.is_cancelling(): - assert self.shutting_down - foreman_logger.debug( - 'Task %s (pk=%d) was successful but run "%s" (Pipeline: %s, User: %s) %s.', - finished_task, - finished_task.pk, - self.sandbox.run, - self.sandbox.pipeline, - self.sandbox.user, - "failing" if self.sandbox.run.is_failing() else "cancelling" - ) - - # Stop any sub-Runs (or the top-level run) that this was the last - # running task of. - stop_subruns_if_possible = True - - if not tasks_currently_running: - clean_up_now = True - - else: # run is still processing successfully - # Was this task a recovery or novel progress? - if task_execute_info.is_recovery(): - foreman_logger.debug( - 'Recovering task %s (pk=%d) was successful; ' - 'queueing waiting tasks from run "%s" (Pipeline: %s, User: %s).', - finished_task, - finished_task.pk, - self.sandbox.run, - self.sandbox.pipeline, - self.sandbox.user - ) - - execrecordouts = task_execute_info.execrecord.execrecordouts.all() - data_newly_available = [execrecordout.dataset - for execrecordout in execrecordouts] - # Add anything that was waiting on this recovery to the queue. - self.sandbox.enqueue_runnable_tasks(data_newly_available) - - else: - foreman_logger.debug( - 'Task %s (pk=%d) was successful; ' - 'advancing run "%s" (Pipeline: %s, User: %s).', - finished_task, - finished_task.pk, - self.sandbox.run, - self.sandbox.pipeline, - self.sandbox.user - ) - - # Update maps and advance the pipeline. Note that advance_pipeline - # will transition the states of runs and sub-runs appropriately. - self.sandbox.update_sandbox(finished_task) - self.sandbox.advance_pipeline(task_completed=finished_task) - self.sandbox.run.refresh_from_db() - if self.sandbox.run.is_successful(): - clean_up_now = not tasks_currently_running - if clean_up_now and self.sandbox.run.is_successful(): - foreman_logger.info( - 'Run "%s" (Pipeline: %s, User: %s) finished successfully', - self.sandbox.run, - self.sandbox.pipeline, - self.sandbox.user - ) - elif self.sandbox.run.is_failing() or self.sandbox.run.is_cancelling(): - # Something just failed in advance_pipeline. - foreman_logger.debug( - 'Run "%s" (Pipeline: %s, User: %s) failed to advance ' - 'after finishing task %s (pk=%d)', - self.sandbox.run, - self.sandbox.pipeline, - self.sandbox.user, - finished_task, - finished_task.pk - ) - - if not self.shutting_down: - self.mop_up() # this sets self.shutting_down. - clean_up_now = not tasks_currently_running - - else: - # The component that just finished failed or was cancelled (e.g. a RunCable fails to - # copy the input to the sandbox). Cancellation is handled by stop_run - # (or assign_task). - assert finished_task.is_failed() or finished_task.is_cancelled(), "{} != Failed or Cancelled".format( - finished_task.get_state_name() - ) - stop_subruns_if_possible = True - if self.sandbox.run.is_failing() or self.sandbox.run.is_cancelling(): - assert self.shutting_down - foreman_logger.debug( - 'Task %s (pk=%d) %s; run "%s" (Pipeline: %s, User: %s) was already %s', - finished_task, - finished_task.pk, - finished_task.get_state_name(), - self.sandbox.run, - self.sandbox.pipeline, - self.sandbox.user, - "failing" if self.sandbox.run.is_failing() else "cancelling" - ) - - else: - assert self.sandbox.run.is_running(), "{} != Running".format(self.sandbox.run.get_state_name()) - foreman_logger.info( - 'Task %s (pk=%d) of run "%s" (Pipeline: %s, User: %s) failed; ' - 'marking run as failing', - finished_task, - finished_task.pk, - self.sandbox.run, - self.sandbox.pipeline, - self.sandbox.user - ) - - # Go through and mark all ancestor runs of finished_task as failing. - self.sandbox.run.mark_failure(save=True) - curr_ancestor_run = self.sandbox.run - # This does nothing if finished_task_coords is of length 1; i.e. if it's a component belonging - # to the top-level run. - for coord in finished_task_coords[:-1]: - curr_ancestor_run = curr_ancestor_run.runsteps.get(pipelinestep__step_num=coord).child_run - if curr_ancestor_run.is_running(): # skip over this if it's cancelling or failing already - curr_ancestor_run.mark_failure(save=True) - - self.mop_up() # this cancels the recovering record as well - - # Now check whether we can do our final clean up. - if not tasks_currently_running: - clean_up_now = True - - if stop_subruns_if_possible: - - curr_run = finished_task.parent_run - for idx in range(len(finished_task_coords)-1, -1, -1): - # finished_task_coords[idx] is the component coordinate in - # the subrun idx levels deep (0 means top-level run). - # curr_run is this subrun. - curr_run.refresh_from_db() - if not subrun_tasks_currently_running[idx]: - if not curr_run.has_ended(): - curr_run.stop(save=True) - else: - curr_run.finish_recovery(save=True) - else: - # By definition everything above this point has stuff - # still running. - break - if idx > 0: - curr_run = curr_run.parent_runstep.parent_run - - if not clean_up_now: - # The Run is still going and there may be more stuff to do. - self.submit_ready_tasks() - - else: - self.sandbox.run.refresh_from_db() - foreman_logger.info( - 'Cleaning up %s run "%s" (Pipeline: %s, User: %s)', - self.sandbox.run.get_state_name(), - self.sandbox.run, - self.sandbox.pipeline, - self.sandbox.user - ) - - # Having reached here, this run should have been properly stopped in the - # "if stop_subruns_if_possible" block. - assert self.sandbox.run.is_complete(), \ - self.sandbox.run.get_state_name() + " is not one of the complete states" - - def stop_run(self): - """ - Stop this Foreman's run. - """ - mgr_logger.debug("Stopping run (pk=%d) on behalf of user %s", - self.sandbox.run.pk, - self.sandbox.run.stopped_by) - - if not self.sandbox.run.has_started(): - self.sandbox.run.start(save=True) - - if self.sandbox.run.is_complete(): - # This run already completed, so we ignore this call. - mgr_logger.warn("Run (pk=%d) is already complete; ignoring stop request.", - self.sandbox.run.pk) - return - - else: - self.cancel_all_slurm_jobs() - self.tasks_in_progress.clear() - # Mark all RunComponents as cancelled, and finalize the - # details. - self.mop_up() - - if self.sandbox.run.is_running(): - # No tasks are running now so there is nothing that could screw up the Run state - # between the previous line and this line. - self.sandbox.run.cancel(save=True) - self.sandbox.run.stop(save=True) - - foreman_logger.debug( - "Run (pk={}) stopped by user {}".format( - self.sandbox.run.pk, - self.sandbox.run.stopped_by - ) - ) - - def cancel_all_slurm_jobs(self): - """ - Cancel all Slurm jobs relating to this Foreman. - """ - for task, task_dict in six.iteritems(self.tasks_in_progress): - if isinstance(task, RunStep): - parts_to_kill = ("setup", "driver") - if "bookkeeping" in task_dict: - parts_to_kill = parts_to_kill + ("bookkeeping",) - for job in parts_to_kill: - self.slurm_sched_class.job_cancel(task_dict[job]) - else: - self.slurm_sched_class.job_cancel(task_dict["cable"]) - - if os.path.exists(task_dict["info_path"]): - os.remove(task_dict["info_path"]) diff --git a/kive/kive/testing_utils.py b/kive/kive/testing_utils.py index bbe0b4f76..9947baa86 100644 --- a/kive/kive/testing_utils.py +++ b/kive/kive/testing_utils.py @@ -1,352 +1,12 @@ -import six -import csv -import os -import random -import shutil -import subprocess -import tempfile -import logging -import hashlib -import time - -from django.conf import settings -from django.contrib.auth.models import User -from django.core.files import File -from django.db import transaction -from django.db.models import Count - -from constants import datatypes -import file_access_utils -from librarian.models import Dataset, ExecRecord -from metadata.models import BasicConstraint, CompoundDatatype, Datatype, everyone_group -from method.models import CodeResource, CodeResourceRevision, Method, MethodFamily -from pipeline.models import Pipeline, PipelineFamily, PipelineStep -from archive.models import RunStep, ExecLog, MethodOutput +from librarian.models import Dataset from datachecking.models import VerificationLog -from fleet.workers import Manager -from fleet.slurmlib import DummySlurmScheduler - - -samplecode_path = "../samplecode" - - -# This is copied from -# http://stackoverflow.com/questions/2023608/check-what-files-are-open-in-python -def get_open_fds(): - """ - Return the number of open file descriptors for the current process. - - Warning: will only work on UNIX-like operating systems. - """ - pid = os.getpid() - procs = subprocess.check_output(["lsof", '-w', '-Ff', "-p", str(pid)]) - procs_list = list( - filter( - lambda s: s and s[0] == 'f' and s[1:].isdigit(), - procs.decode(encoding="utf-8").split('\n') - ) - ) - nprocs = len(procs_list) - return nprocs - - -# For tracking whether we're leaking file descriptors. -fd_count_logger = logging.getLogger("method.tests") - - -def fd_count(msg): - fd_count_logger.debug("{}: {}".format(msg, get_open_fds())) - - -def create_metadata_test_environment(case): - """Setup default database state from which to perform unit testing.""" - # Define a user. - case.myUser = User.objects.create_user('john', - 'lennon@thebeatles.com', - 'johnpassword') - case.myUser.save() - case.ringoUser = User.objects.create_user('ringo', - 'starr@thebeatles.com', - 'ringopassword') - case.ringoUser.save() - case.myUser.groups.add(everyone_group()) - case.myUser.save() - - # Load up the builtin Datatypes. - case.STR = Datatype.objects.get(pk=datatypes.STR_PK) - case.FLOAT = Datatype.objects.get(pk=datatypes.FLOAT_PK) - case.INT = Datatype.objects.get(pk=datatypes.INT_PK) - case.BOOL = Datatype.objects.get(pk=datatypes.BOOL_PK) - - # Many tests use case.string_dt as a name for case.STR. - case.string_dt = case.STR - - # Create Datatype "DNANucSeq" with a regexp basic constraint. - case.DNA_dt = Datatype( - name="DNANucSeq", - description="String consisting of ACGTacgt", - user=case.myUser) - case.DNA_dt.save() - # DNA_dt is a restricted type of string - case.DNA_dt.restricts.add(case.string_dt) - case.DNA_dt.grant_everyone_access() - case.DNA_dt.basic_constraints.create( - ruletype=BasicConstraint.REGEXP, - rule="^[ACGTacgt]*$") - case.DNA_dt.save() - - # Similarly, create Datatype "RNANucSeq". - case.RNA_dt = Datatype( - name="RNANucSeq", - description="String consisting of ACGUacgu", - user=case.myUser) - case.RNA_dt.save() - # RNA_dt is a restricted type of string - case.RNA_dt.restricts.add(case.string_dt) - case.RNA_dt.grant_everyone_access() - case.RNA_dt.basic_constraints.create( - ruletype=BasicConstraint.REGEXP, - rule="^[ACGUacgu]*$") - case.RNA_dt.save() - - # Define a new CDT with a bunch of different member - case.basic_cdt = CompoundDatatype(user=case.myUser) - case.basic_cdt.save() - case.basic_cdt.grant_everyone_access() - case.basic_cdt.save() - - case.basic_cdt.members.create( - datatype=case.string_dt, - column_name='label', - column_idx=1) - case.basic_cdt.members.create( - datatype=case.INT, - column_name='integer', - column_idx=2) - case.basic_cdt.members.create( - datatype=case.FLOAT, - column_name='float', - column_idx=3) - case.basic_cdt.members.create( - datatype=case.BOOL, - column_name='bool', - column_idx=4) - case.basic_cdt.members.create( - datatype=case.RNA_dt, - column_name="rna", - column_idx=5) - case.basic_cdt.full_clean() - case.basic_cdt.save() - - # Define a new CDT that is only accessible to two users - shared_cdt = CompoundDatatype(user=case.myUser) - shared_cdt.save() - shared_cdt.users_allowed.add(case.ringoUser) - shared_cdt.save() - - shared_cdt.members.create( - datatype=case.string_dt, - column_name='label', - column_idx=1) - - # Define test_cdt as containing 3 members: - # (label, PBMCseq, PLAseq) as (string,DNA,RNA) - case.test_cdt = CompoundDatatype(user=case.myUser) - case.test_cdt.save() - case.test_cdt.grant_everyone_access() - case.test_cdt.save() - case.test_cdt.members.create( - datatype=case.string_dt, - column_name="label", - column_idx=1) - case.test_cdt.members.create( - datatype=case.DNA_dt, - column_name="PBMCseq", - column_idx=2) - case.test_cdt.members.create( - datatype=case.RNA_dt, - column_name="PLAseq", - column_idx=3) - case.test_cdt.full_clean() - case.test_cdt.save() - - # Define DNAinput_cdt (1 member) - case.DNAinput_cdt = CompoundDatatype(user=case.myUser) - case.DNAinput_cdt.save() - case.DNAinput_cdt.members.create( - datatype=case.DNA_dt, - column_name="SeqToComplement", - column_idx=1) - case.DNAinput_cdt.grant_everyone_access() - case.DNAinput_cdt.full_clean() - case.DNAinput_cdt.save() - - # Define DNAoutput_cdt (1 member) - case.DNAoutput_cdt = CompoundDatatype(user=case.myUser) - case.DNAoutput_cdt.save() - case.DNAoutput_cdt.members.create( - datatype=case.DNA_dt, - column_name="ComplementedSeq", - column_idx=1) - case.DNAoutput_cdt.grant_everyone_access() - case.DNAoutput_cdt.full_clean() - case.DNAoutput_cdt.save() - - # Define RNAinput_cdt (1 column) - case.RNAinput_cdt = CompoundDatatype(user=case.myUser) - case.RNAinput_cdt.save() - case.RNAinput_cdt.members.create( - datatype=case.RNA_dt, - column_name="SeqToComplement", - column_idx=1) - case.RNAinput_cdt.grant_everyone_access() - case.RNAinput_cdt.full_clean() - case.RNAinput_cdt.save() - - # Define RNAoutput_cdt (1 column) - case.RNAoutput_cdt = CompoundDatatype(user=case.myUser) - case.RNAoutput_cdt.save() - case.RNAoutput_cdt.members.create( - datatype=case.RNA_dt, - column_name="ComplementedSeq", - column_idx=1) - case.RNAoutput_cdt.grant_everyone_access() - case.RNAoutput_cdt.full_clean() - case.RNAoutput_cdt.save() - - #### - # Everything above this point is used in metadata.tests. - # This next bit is used in method.tests. - - # Define "tuple" CDT containing (x,y): members x and y exist at index 1 and 2 - case.tuple_cdt = CompoundDatatype(user=case.myUser) - case.tuple_cdt.save() - case.tuple_cdt.members.create(datatype=case.string_dt, column_name="x", column_idx=1) - case.tuple_cdt.members.create(datatype=case.string_dt, column_name="y", column_idx=2) - case.tuple_cdt.grant_everyone_access() - - # Define "singlet" CDT containing CDT member (a) and "triplet" CDT with members (a,b,c) - case.singlet_cdt = CompoundDatatype(user=case.myUser) - case.singlet_cdt.save() - case.singlet_cdt.members.create( - datatype=case.string_dt, column_name="k", column_idx=1) - case.singlet_cdt.grant_everyone_access() - - case.triplet_cdt = CompoundDatatype(user=case.myUser) - case.triplet_cdt.save() - case.triplet_cdt.members.create(datatype=case.string_dt, column_name="a", column_idx=1) - case.triplet_cdt.members.create(datatype=case.string_dt, column_name="b", column_idx=2) - case.triplet_cdt.members.create(datatype=case.string_dt, column_name="c", column_idx=3) - case.triplet_cdt.grant_everyone_access() - - #### - # This next bit is used for pipeline.tests. - - # Define CDT "triplet_squares_cdt" with 3 members for use as an input/output - case.triplet_squares_cdt = CompoundDatatype(user=case.myUser) - case.triplet_squares_cdt.save() - case.triplet_squares_cdt.members.create(datatype=case.string_dt, column_name="a^2", column_idx=1) - case.triplet_squares_cdt.members.create(datatype=case.string_dt, column_name="b^2", column_idx=2) - case.triplet_squares_cdt.members.create(datatype=case.string_dt, column_name="c^2", column_idx=3) - case.triplet_squares_cdt.grant_everyone_access() - - # A CDT with mixed Datatypes - case.mix_triplet_cdt = CompoundDatatype(user=case.myUser) - case.mix_triplet_cdt.save() - case.mix_triplet_cdt.members.create(datatype=case.string_dt, column_name="StrCol1", column_idx=1) - case.mix_triplet_cdt.members.create(datatype=case.DNA_dt, column_name="DNACol2", column_idx=2) - case.mix_triplet_cdt.members.create(datatype=case.string_dt, column_name="StrCol3", column_idx=3) - case.mix_triplet_cdt.grant_everyone_access() - - # Define CDT "doublet_cdt" same as tuple: x, y - case.doublet_cdt = case.tuple_cdt - - #### - # Stuff from this point on is used in librarian and archive - # testing. - - # October 15: more CDTs. - case.DNA_triplet_cdt = CompoundDatatype(user=case.myUser) - case.DNA_triplet_cdt.save() - case.DNA_triplet_cdt.members.create(datatype=case.DNA_dt, column_name="a", column_idx=1) - case.DNA_triplet_cdt.members.create(datatype=case.DNA_dt, column_name="b", column_idx=2) - case.DNA_triplet_cdt.members.create(datatype=case.DNA_dt, column_name="c", column_idx=3) - case.DNA_triplet_cdt.grant_everyone_access() - - case.DNA_doublet_cdt = CompoundDatatype(user=case.myUser) - case.DNA_doublet_cdt.save() - case.DNA_doublet_cdt.members.create(datatype=case.DNA_dt, column_name="x", column_idx=1) - case.DNA_doublet_cdt.members.create(datatype=case.DNA_dt, column_name="y", column_idx=2) - case.DNA_doublet_cdt.grant_everyone_access() - - -def load_metadata_test_environment(case): - case.myUser = User.objects.get(username='john') - case.ringoUser = User.objects.get(username='ringo') - - case.STR = Datatype.objects.get(pk=datatypes.STR_PK) - case.FLOAT = Datatype.objects.get(pk=datatypes.FLOAT_PK) - case.INT = Datatype.objects.get(pk=datatypes.INT_PK) - case.BOOL = Datatype.objects.get(pk=datatypes.BOOL_PK) - case.string_dt = case.STR - - case.DNA_dt = Datatype.objects.get(name="DNANucSeq") - case.RNA_dt = Datatype.objects.get(name="RNANucSeq") - - case.basic_cdt = CompoundDatatype.objects.get(members__column_name='rna') - counted = CompoundDatatype.objects.annotate(num_members=Count('members')) - case.shared_cdt = counted.get(members__column_name='label', - num_members=1) - case.test_cdt = CompoundDatatype.objects.get(members__column_name='PBMCseq') - case.DNAinput_cdt = CompoundDatatype.objects.get( - members__datatype=case.DNA_dt, - members__column_name='SeqToComplement') - case.DNAoutput_cdt = CompoundDatatype.objects.get( - members__datatype=case.DNA_dt, - members__column_name='ComplementedSeq') - case.RNAinput_cdt = CompoundDatatype.objects.get( - members__datatype=case.RNA_dt, - members__column_name='SeqToComplement') - case.RNAoutput_cdt = CompoundDatatype.objects.get( - members__datatype=case.RNA_dt, - members__column_name='ComplementedSeq') - case.tuple_cdt = CompoundDatatype.objects.get( - members__datatype=case.string_dt, - members__column_name='x') - case.singlet_cdt = CompoundDatatype.objects.get(members__column_name='k') - case.triplet_cdt = CompoundDatatype.objects.get( - members__datatype=case.string_dt, - members__column_name='a') - case.triplet_squares_cdt = CompoundDatatype.objects.get( - members__column_name='a^2') - case.mix_triplet_cdt = CompoundDatatype.objects.get( - members__column_name='StrCol1') - case.doublet_cdt = case.tuple_cdt - case.DNA_triplet_cdt = CompoundDatatype.objects.get( - members__datatype=case.DNA_dt, - members__column_name='a') - case.DNA_doublet_cdt = CompoundDatatype.objects.get( - members__datatype=case.DNA_dt, - members__column_name='x') def clean_up_all_files(): """ Delete all files that have been put into the database as FileFields. """ - for crr in CodeResourceRevision.objects.all(): - # Remember that this can be empty. - # if crr.content_file != None: - # crr.content_file.delete() - # Weirdly, if crr.content_file == None, - # it still entered the above. This seems to be a bug - # in Django! - if crr.coderesource.filename != "": - crr.content_file.close() - crr.content_file.delete() - - crr.delete() # Also clear all datasets. This was previously in librarian.tests # but we move it here. @@ -355,1999 +15,9 @@ def clean_up_all_files(): dataset.dataset_file.delete() dataset.delete() - for mo in MethodOutput.objects.all(): - mo.output_log.close() - mo.output_log.delete() - mo.error_log.close() - mo.error_log.delete() - mo.delete() - for vl in VerificationLog.objects.all(): vl.output_log.close() vl.output_log.delete() vl.error_log.close() vl.error_log.delete() vl.delete() - - -def create_eric_martin_test_environment(case): - """ - Set up the original test state Eric Martin designed. - - This sets up the environment as in the Metadata tests, and then augments with - Methods, CR/CRR/CRDs, and DT/CDTs. Note that these are *not* the same - as those set up in the Method testing. - """ - create_metadata_test_environment(case) - - #### - # This is the big pipeline Eric developed that was originally - # used in copperfish/tests. - # CRs and CRRs: - case.generic_cr = CodeResource( - name="genericCR", description="Just a CR", - filename="generic_script.py", user=case.myUser) - case.generic_cr.save() - case.generic_cr.grant_everyone_access() - with open(os.path.join(samplecode_path, "generic_script.py"), "rb") as f: - case.generic_crRev = CodeResourceRevision( - coderesource=case.generic_cr, - revision_name="v1", - revision_desc="desc", - user=case.myUser, - content_file=File(f) - ) - case.generic_crRev.clean() - case.generic_crRev.save() - case.generic_crRev.grant_everyone_access() - - # Method family, methods, and their input/outputs - case.mf = MethodFamily(name="method_family", - description="Holds methods A/B/C", - user=case.myUser) - case.mf.save() - case.mf.grant_everyone_access() - case.mA = Method(revision_name="mA_name", revision_desc="A_desc", family=case.mf, driver=case.generic_crRev, - user=case.myUser) - case.mA.save() - case.mA.grant_everyone_access() - case.A1_rawin = case.mA.create_input(dataset_name="A1_rawin", dataset_idx=1) - case.A1_out = case.mA.create_output(compounddatatype=case.doublet_cdt, - dataset_name="A1_out", - dataset_idx=1) - - case.mB = Method(revision_name="mB_name", revision_desc="B_desc", family=case.mf, driver=case.generic_crRev, - user=case.myUser) - case.mB.save() - case.mB.grant_everyone_access() - case.B1_in = case.mB.create_input(compounddatatype=case.doublet_cdt, - dataset_name="B1_in", - dataset_idx=1) - case.B2_in = case.mB.create_input(compounddatatype=case.singlet_cdt, - dataset_name="B2_in", - dataset_idx=2) - case.B1_out = case.mB.create_output(compounddatatype=case.triplet_cdt, - dataset_name="B1_out", - dataset_idx=1, - max_row=5) - - case.mC = Method(revision_name="mC_name", revision_desc="C_desc", family=case.mf, driver=case.generic_crRev, - user=case.myUser) - case.mC.save() - case.mC.grant_everyone_access() - case.C1_in = case.mC.create_input(compounddatatype=case.triplet_cdt, - dataset_name="C1_in", - dataset_idx=1) - case.C2_in = case.mC.create_input(compounddatatype=case.doublet_cdt, - dataset_name="C2_in", - dataset_idx=2) - case.C1_out = case.mC.create_output(compounddatatype=case.singlet_cdt, - dataset_name="C1_out", - dataset_idx=1) - case.C2_rawout = case.mC.create_output(dataset_name="C2_rawout", - dataset_idx=2) - case.C3_rawout = case.mC.create_output(dataset_name="C3_rawout", - dataset_idx=3) - - # Pipeline family, pipelines, and their input/outputs - case.pf = PipelineFamily(name="Pipeline_family", description="PF desc", user=case.myUser) - case.pf.save() - case.pf.grant_everyone_access() - case.pD = Pipeline(family=case.pf, revision_name="pD_name", revision_desc="D", user=case.myUser) - case.pD.save() - case.pD.grant_everyone_access() - case.D1_in = case.pD.create_input(compounddatatype=case.doublet_cdt, - dataset_name="D1_in", - dataset_idx=1) - case.D2_in = case.pD.create_input(compounddatatype=case.singlet_cdt, - dataset_name="D2_in", - dataset_idx=2) - case.pE = Pipeline(family=case.pf, revision_name="pE_name", revision_desc="E", user=case.myUser) - case.pE.save() - case.pE.grant_everyone_access() - case.E1_in = case.pE.create_input(compounddatatype=case.triplet_cdt, - dataset_name="E1_in", - dataset_idx=1) - case.E2_in = case.pE.create_input(compounddatatype=case.singlet_cdt, - dataset_name="E2_in", - dataset_idx=2, - min_row=10) - case.E3_rawin = case.pE.create_input(dataset_name="E3_rawin", - dataset_idx=3) - - # Pipeline steps - case.step_D1 = case.pD.steps.create(transformation=case.mB, - step_num=1) - case.step_E1 = case.pE.steps.create(transformation=case.mA, - step_num=1) - case.step_E2 = case.pE.steps.create(transformation=case.pD, - step_num=2) - case.step_E3 = case.pE.steps.create(transformation=case.mC, - step_num=3) - - # Pipeline cables and outcables - case.D01_11 = case.step_D1.cables_in.create(dest=case.B1_in, - source_step=0, - source=case.D1_in) - case.D02_12 = case.step_D1.cables_in.create(dest=case.B2_in, - source_step=0, - source=case.D2_in) - case.D11_21 = case.pD.outcables.create(output_name="D1_out", - output_idx=1, - output_cdt=case.triplet_cdt, - source_step=1, - source=case.B1_out) - case.pD.create_outputs() - case.D1_out = case.pD.outputs.get(dataset_name="D1_out") - - case.E03_11 = case.step_E1.cables_in.create(dest=case.A1_rawin, - source_step=0, - source=case.E3_rawin) - case.E01_21 = case.step_E2.cables_in.create(dest=case.D1_in, - source_step=0, - source=case.E1_in) - case.E02_22 = case.step_E2.cables_in.create(dest=case.D2_in, - source_step=0, - source=case.E2_in) - case.E11_32 = case.step_E3.cables_in.create(dest=case.C2_in, - source_step=1, - source=case.A1_out) - case.E21_31 = case.step_E3.cables_in.create( - dest=case.C1_in, - source_step=2, - source=case.step_E2.transformation.outputs.get(dataset_name="D1_out")) - case.E21_41 = case.pE.outcables.create( - output_name="E1_out", - output_idx=1, - output_cdt=case.doublet_cdt, - source_step=2, - source=case.step_E2.transformation.outputs.get(dataset_name="D1_out")) - case.E31_42 = case.pE.outcables.create(output_name="E2_out", - output_idx=2, - output_cdt=case.singlet_cdt, - source_step=3, - source=case.C1_out) - case.E33_43 = case.pE.outcables.create(output_name="E3_rawout", - output_idx=3, - output_cdt=None, - source_step=3, - source=case.C3_rawout) - case.pE.create_outputs() - case.E1_out = case.pE.outputs.get(dataset_name="E1_out") - case.E2_out = case.pE.outputs.get(dataset_name="E2_out") - case.E3_rawout = case.pE.outputs.get(dataset_name="E3_rawout") - - # Custom wiring/outwiring - case.E01_21_wire1 = case.E01_21.custom_wires.create( - source_pin=case.triplet_cdt.members.get(column_idx=1), dest_pin=case.doublet_cdt.members.get(column_idx=2)) - case.E01_21_wire2 = case.E01_21.custom_wires.create( - source_pin=case.triplet_cdt.members.get(column_idx=3), dest_pin=case.doublet_cdt.members.get(column_idx=1)) - case.E11_32_wire1 = case.E11_32.custom_wires.create( - source_pin=case.doublet_cdt.members.get(column_idx=1), dest_pin=case.doublet_cdt.members.get(column_idx=2)) - case.E11_32_wire2 = case.E11_32.custom_wires.create( - source_pin=case.doublet_cdt.members.get(column_idx=2), dest_pin=case.doublet_cdt.members.get(column_idx=1)) - case.E21_41_wire1 = case.E21_41.custom_wires.create( - source_pin=case.triplet_cdt.members.get(column_idx=2), dest_pin=case.doublet_cdt.members.get(column_idx=2)) - case.E21_41_wire2 = case.E21_41.custom_wires.create( - source_pin=case.triplet_cdt.members.get(column_idx=3), dest_pin=case.doublet_cdt.members.get(column_idx=1)) - case.pE.clean() - - # November 7, 2013: use a helper function (defined in - # librarian.models) to define our Datasets. - - # Define singlet, doublet, triplet, and raw uploaded datasets - case.triplet_dataset = Dataset.create_dataset( - os.path.join(samplecode_path, "step_0_triplet.csv"), - case.myUser, - groups_allowed=[everyone_group()], - cdt=case.triplet_cdt, - keep_file=True, - name="triplet", - description="lol" - ) - case.triplet_dataset_structure = case.triplet_dataset.structure - - case.doublet_dataset = Dataset.create_dataset( - os.path.join(samplecode_path, "doublet_cdt.csv"), - case.myUser, - groups_allowed=[everyone_group()], - cdt=case.doublet_cdt, - name="doublet", - description="lol" - ) - case.doublet_dataset_structure = case.doublet_dataset.structure - - case.singlet_dataset = Dataset.create_dataset( - os.path.join(samplecode_path, "singlet_cdt_large.csv"), - case.myUser, - groups_allowed=[everyone_group()], - cdt=case.singlet_cdt, - name="singlet", - description="lol" - ) - case.singlet_dataset_structure = case.singlet_dataset.structure - - case.singlet_3rows_dataset = Dataset.create_dataset( - os.path.join(samplecode_path, "step_0_singlet.csv"), - case.myUser, - groups_allowed=[everyone_group()], - cdt=case.singlet_cdt, - name="singlet", - description="lol" - ) - case.singlet_3rows_dataset_structure = case.singlet_3rows_dataset.structure - - case.raw_dataset = Dataset.create_dataset( - os.path.join(samplecode_path, "step_0_raw.fasta"), - user=case.myUser, - groups_allowed=[everyone_group()], - cdt=None, - name="raw_DS", - description="lol" - ) - - # Added September 30, 2013: dataset that results from E01_21. - # November 7, 2013: created a file that this Dataset actually represented, - # even though it isn't in the database. - case.D1_in_dataset = Dataset.create_dataset( - os.path.join(samplecode_path, "doublet_remuxed_from_triplet.csv"), - user=case.myUser, - groups_allowed=[everyone_group()], - cdt=case.doublet_cdt, - name="D1_in_doublet", - keep_file=False - ) - case.D1_in_dataset_structure = case.D1_in_dataset.structure - - case.C1_in_dataset = Dataset.create_dataset( - os.path.join(samplecode_path, "C1_in_triplet.csv"), - case.myUser, - groups_allowed=[everyone_group()], - cdt=case.triplet_cdt, - name="C1_in_triplet", - description="triplet 3 rows" - ) - case.C1_in_dataset_structure = case.C1_in_dataset.structure - - # November 7, 2013: compute the MD5 checksum from the data file, - # which is the same as below. - case.C2_in_dataset = Dataset.create_dataset( - os.path.join(samplecode_path, "E11_32_output.csv"), - case.myUser, - groups_allowed=[everyone_group()], - cdt=case.doublet_cdt, - keep_file=False, - name="C2_in_doublet" - ) - case.C2_in_dataset_structure = case.C2_in_dataset.structure - - # October 16: an alternative to C2_in_dataset, which has existent data. - case.E11_32_output_dataset = Dataset.create_dataset( - os.path.join(samplecode_path, "E11_32_output.csv"), - case.myUser, - groups_allowed=[everyone_group()], - cdt=case.doublet_cdt, - name="E11_32 output doublet", - description="result of E11_32 fed by doublet_cdt.csv" - ) - case.E11_32_output_dataset_structure = case.E11_32_output_dataset.structure - - case.C1_out_dataset = Dataset.create_dataset( - os.path.join(samplecode_path, "step_0_singlet.csv"), - case.myUser, - groups_allowed=[everyone_group()], - cdt=case.singlet_cdt, - name="raw", - description="lol" - ) - case.C1_out_dataset_structure = case.C1_out_dataset.structure - - case.C2_out_dataset = Dataset.create_dataset( - os.path.join(samplecode_path, "step_0_raw.fasta"), - case.myUser, - groups_allowed=[everyone_group()], - cdt=None, - name="C2_out", - description="lol" - ) - - case.C3_out_dataset = Dataset.create_dataset( - os.path.join(samplecode_path, "step_0_raw.fasta"), - case.myUser, - groups_allowed=[everyone_group()], - cdt=None, - name="C3_out", - description="lol" - ) - - case.triplet_3_rows_dataset = Dataset.create_dataset( - os.path.join(samplecode_path, "step_0_triplet_3_rows.csv"), - case.myUser, - groups_allowed=[everyone_group()], - cdt=case.triplet_cdt, - name="triplet", - description="lol" - ) - case.triplet_3_rows_dataset_structure = case.triplet_3_rows_dataset.structure - - # October 9, 2013: added as the result of cable E21_41. - case.E1_out_dataset = Dataset.create_dataset( - os.path.join(samplecode_path, "doublet_remuxed_from_t3r.csv"), - case.myUser, - groups_allowed=[everyone_group()], - cdt=case.doublet_cdt, - name="E1_out", - description="doublet remuxed from triplet" - ) - case.E1_out_dataset_structure = case.E1_out_dataset.structure - - # October 15, 2013: Datasets that go into and come out - # of cable E01_21 and E21_41. - case.DNA_triplet_dataset = Dataset.create_dataset( - os.path.join(samplecode_path, "DNA_triplet.csv"), - case.myUser, - groups_allowed=[everyone_group()], - cdt=case.DNA_triplet_cdt, - name="DNA_triplet", - description="DNA triplet data" - ) - case.DNA_triplet_dataset_structure = case.DNA_triplet_dataset.structure - - case.E01_21_DNA_doublet_dataset = Dataset.create_dataset( - os.path.join(samplecode_path, "E01_21_DNA_doublet.csv"), - case.myUser, - groups_allowed=[everyone_group()], - cdt=case.DNA_doublet_cdt, - name="E01_21_DNA_doublet", - description="DNA doublet data coming from DNA_triplet.csv but remultiplexed according to cable E01_21" - ) - case.E01_21_DNA_doublet_dataset_structure = case.E01_21_DNA_doublet_dataset.structure - - case.E21_41_DNA_doublet_dataset = Dataset.create_dataset( - os.path.join(samplecode_path, "E21_41_DNA_doublet.csv"), - case.myUser, - groups_allowed=[everyone_group()], - cdt=case.DNA_doublet_cdt, - name="E21_41_DNA_doublet", - description="DNA doublet data coming from DNA_triplet.csv but remultiplexed according to cable E21_41" - ) - case.E21_41_DNA_doublet_dataset_structure = case.E21_41_DNA_doublet_dataset.structure - - -def load_eric_martin_test_environment(case): - load_metadata_test_environment(case) - - case.generic_cr = CodeResource.objects.get(name="genericCR") - case.generic_crRev = case.generic_cr.revisions.get() - - case.mf = MethodFamily.objects.get(name="method_family") - case.mA = Method.objects.get(revision_name="mA_name") - case.A1_rawin = case.mA.inputs.get() - case.A1_out = case.mA.outputs.get() - - case.mB = Method.objects.get(revision_name="mB_name") - case.B1_in = case.mB.inputs.get(dataset_name="B1_in") - case.B2_in = case.mB.inputs.get(dataset_name="B2_in") - case.B1_out = case.mB.outputs.get() - - case.mC = Method.objects.get(revision_name="mC_name") - case.C1_in = case.mC.inputs.get(dataset_name="C1_in") - case.C2_in = case.mC.inputs.get(dataset_name="C2_in") - case.C1_out = case.mC.outputs.get(dataset_name="C1_out") - case.C2_rawout = case.mC.outputs.get(dataset_name="C2_rawout") - case.C3_rawout = case.mC.outputs.get(dataset_name="C3_rawout") - - case.pf = PipelineFamily.objects.get(name="Pipeline_family") - case.pD = Pipeline.objects.get(revision_name="pD_name") - case.D1_in = case.pD.inputs.get(dataset_name="D1_in") - case.D2_in = case.pD.inputs.get(dataset_name="D2_in") - case.pE = Pipeline.objects.get(revision_name="pE_name") - case.E1_in = case.pE.inputs.get(dataset_name="E1_in") - case.E2_in = case.pE.inputs.get(dataset_name="E2_in") - case.E3_rawin = case.pE.inputs.get(dataset_name="E3_rawin") - - case.step_D1 = case.pD.steps.get(step_num=1) - case.step_E1 = case.pE.steps.get(step_num=1) - case.step_E2 = case.pE.steps.get(step_num=2) - case.step_E3 = case.pE.steps.get(step_num=3) - - case.D01_11 = case.step_D1.cables_in.get(dest=case.B1_in) - case.D02_12 = case.step_D1.cables_in.get(dest=case.B2_in) - case.D11_21 = case.pD.outcables.get(output_name="D1_out") - case.D1_out = case.pD.outputs.get(dataset_name="D1_out") - - case.E03_11 = case.step_E1.cables_in.get(dest=case.A1_rawin) - case.E01_21 = case.step_E2.cables_in.get(dest=case.D1_in) - case.E02_22 = case.step_E2.cables_in.get(dest=case.D2_in) - case.E11_32 = case.step_E3.cables_in.get(dest=case.C2_in) - case.E21_31 = case.step_E3.cables_in.get(dest=case.C1_in) - case.E21_41 = case.pE.outcables.get(output_name="E1_out") - case.E31_42 = case.pE.outcables.get(output_name="E2_out") - case.E33_43 = case.pE.outcables.get(output_name="E3_rawout") - case.E1_out = case.pE.outputs.get(dataset_name="E1_out") - case.E2_out = case.pE.outputs.get(dataset_name="E2_out") - case.E3_rawout = case.pE.outputs.get(dataset_name="E3_rawout") - - case.E01_21_wire1 = case.E01_21.custom_wires.get( - source_pin=case.triplet_cdt.members.get(column_idx=1)) - case.E01_21_wire2 = case.E01_21.custom_wires.get( - source_pin=case.triplet_cdt.members.get(column_idx=3)) - case.E11_32_wire1 = case.E11_32.custom_wires.get( - source_pin=case.doublet_cdt.members.get(column_idx=1)) - case.E11_32_wire2 = case.E11_32.custom_wires.get( - source_pin=case.doublet_cdt.members.get(column_idx=2)) - case.E21_41_wire1 = case.E21_41.custom_wires.get( - source_pin=case.triplet_cdt.members.get(column_idx=2)) - case.E21_41_wire2 = case.E21_41.custom_wires.get( - source_pin=case.triplet_cdt.members.get(column_idx=3)) - - case.triplet_dataset = Dataset.objects.get( - name="triplet", - dataset_file__endswith="step_0_triplet.csv") - case.triplet_dataset_structure = case.triplet_dataset.structure - case.doublet_dataset = Dataset.objects.get(name="doublet") - case.doublet_dataset_structure = case.doublet_dataset.structure - case.singlet_dataset = Dataset.objects.get( - name="singlet", - dataset_file__endswith="singlet_cdt_large.csv") - case.singlet_dataset_structure = case.singlet_dataset.structure - case.singlet_3rows_dataset = Dataset.objects.get( - name="singlet", - dataset_file__endswith="step_0_singlet.csv") - case.singlet_3rows_dataset_structure = case.singlet_3rows_dataset.structure - case.raw_dataset = Dataset.objects.get(name="raw_DS") - - # MD5 calculated on doublet_remuxed_from_triplet.csv file. - case.D1_in_dataset = Dataset.objects.get(name="D1_in_doublet") - case.D1_in_dataset_structure = case.D1_in_dataset.structure - - case.C1_in_dataset = Dataset.objects.get( - name="C1_in_triplet") - case.C1_in_dataset_structure = case.C1_in_dataset.structure - - case.C2_in_dataset = Dataset.objects.get( - name="C2_in_doublet" - ) - case.C2_in_dataset_structure = case.C2_in_dataset.structure - case.E11_32_output_dataset = Dataset.objects.get( - name="E11_32 output doublet") - case.E11_32_output_dataset_structure = case.E11_32_output_dataset.structure - case.C1_out_dataset = Dataset.objects.get(name="raw") - case.C1_out_dataset_structure = case.C1_out_dataset.structure - case.C2_out_dataset = Dataset.objects.get(name="C2_out") - case.C3_out_dataset = Dataset.objects.get(name="C3_out") - - case.triplet_3_rows_dataset = Dataset.objects.get( - name="triplet", - dataset_file__endswith="step_0_triplet_3_rows.csv") - case.triplet_3_rows_dataset_structure = case.triplet_3_rows_dataset.structure - case.E1_out_dataset = Dataset.objects.get(name="E1_out") - case.E1_out_dataset_structure = case.E1_out_dataset.structure - case.DNA_triplet_dataset = Dataset.objects.get( - name="DNA_triplet") - case.DNA_triplet_dataset_structure = case.DNA_triplet_dataset.structure - case.E01_21_DNA_doublet_dataset = Dataset.objects.get( - name="E01_21_DNA_doublet") - case.E01_21_DNA_doublet_dataset_structure = case.E01_21_DNA_doublet_dataset.structure - case.E21_41_DNA_doublet_dataset = Dataset.objects.get( - name="E21_41_DNA_doublet") - case.E21_41_DNA_doublet_dataset_structure = case.E21_41_DNA_doublet_dataset.structure - - -def create_librarian_test_environment(case): - """ - Set up default state for Librarian unit testing. - """ - create_eric_martin_test_environment(case) - - # Runs for the pipelines. - case.pD_run = case.pD.pipeline_instances.create(user=case.myUser, - name='pD_run') - case.pD_run.save() - case.pD_run.grant_everyone_access() - case.pE_run = case.pE.pipeline_instances.create(user=case.myUser, - name='pE_run') - case.pE_run.save() - case.pE_run.grant_everyone_access() - case.pE_run.inputs.create(dataset=case.triplet_dataset, index=1) - case.pE_run.inputs.create(dataset=case.singlet_dataset, index=2) - case.pE_run.inputs.create(dataset=case.raw_dataset, index=3) - - # Some ExecRecords, some failed, others not. - i = 0 - for step in PipelineStep.objects.all(): - if step.is_subpipeline(): - continue - run = step.pipeline.pipeline_instances.create(user=step.pipeline.user) - run.save() - runstep = RunStep(pipelinestep=step, run=run, reused=False) - runstep.save() - execlog = ExecLog.create(runstep, runstep) - execlog.methodoutput.return_code = i % 2 - execlog.methodoutput.save() - execrecord = ExecRecord(generator=execlog) - execrecord.save() - for step_input in step.transformation.inputs.all(): - dataset = Dataset.filter_by_user(step.pipeline.user).filter( - structure__compounddatatype=step_input.compounddatatype).first() - execrecord.execrecordins.create(dataset=dataset, generic_input=step_input) - runstep.execrecord = execrecord - runstep.save() - i += 1 - - -def load_librarian_test_environment(case): - load_eric_martin_test_environment(case) - case.pD_run = case.pD.pipeline_instances.get(name='pD_run') - case.pE_run = case.pE.pipeline_instances.get(name='pE_run') - - -def create_removal_test_environment(): - # We need: - # - a CodeResource with revisions - # - a CodeResourceRevision with dependencies - # - a Datatype - # - a CDT using that Datatype - # - a Dataset with that CDT - # - a Method using that CDT - # - a Pipeline containing that Method - # - two Runs from that pipeline, the second reusing the first - remover = User.objects.create_user("RemOver", "rem@over.sucks", "baleeted") - remover.save() - remover.groups.add(everyone_group()) - remover.save() - - noop = make_first_revision( - "Noop", - "A noop script that simply writes its input to its output.", - "noop.bash", - """#!/bin/bash -cat "$1" > "$2" -""", - remover, - grant_everyone_access=False - ) - - # A toy Datatype. - nucleotide_seq = new_datatype("Nucleotide sequence", "Sequences of A, C, G, and T", - Datatype.objects.get(pk=datatypes.STR_PK), - remover, - grant_everyone_access=False) - one_col_nuc_seq = CompoundDatatype(user=remover) - one_col_nuc_seq.save() - one_col_nuc_seq.members.create(datatype=nucleotide_seq, column_name="sequence", column_idx=1) - - seq_datafile = tempfile.NamedTemporaryFile(delete=False) - seq_datafile.write("""sequence -ACGT -ATCG -GATTACA -TTCCTCTA -AAAAAAAG -GGGAGTTC -CCCTCCTC - """.encode()) - seq_datafile.close() - seq_dataset = Dataset.create_dataset( - file_path=seq_datafile.name, - user=remover, - cdt=one_col_nuc_seq, - keep_file=True, - name="Removal test data", - description="A dataset for use in the removal test case." - ) - - nuc_seq_noop = make_first_method( - "Noop (nucleotide sequence)", - "A noop on nucleotide sequences", - noop, - remover, - grant_everyone_access=False - ) - simple_method_io(nuc_seq_noop, one_col_nuc_seq, "nuc_seq_in", "nuc_seq_out") - - # Define a method that uses noop as a dependency. - pass_through = make_first_revision( - "Pass Through", "A script that does nothing to its input and passes it through untouched.", - "passthrough.bash", - """#!/bin/bash -./noop.bash "$1" "$2" -""", - remover, - grant_everyone_access=False - ) - raw_pass_through = make_first_method( - "Pass-through (raw)", - "A pass-through on raw data that uses noop as a dependency", - pass_through, - remover, - grant_everyone_access=False - ) - simple_method_io(raw_pass_through, None, "nuc_seq_in", "nuc_seq_out") - raw_pass_through.dependencies.create(requirement=noop) - - noop_pl = make_first_pipeline( - "Nucleotide Sequence Noop", - "A noop pipeline for nucleotide sequences.", - remover, - grant_everyone_access=False - ) - create_linear_pipeline(noop_pl, [nuc_seq_noop], "noop_pipeline_in", "noop_pipeline_out") - - p_nested = make_first_pipeline("Nested pipeline", - "Pipeline with one nested level", - remover, - grant_everyone_access=False) - create_linear_pipeline(p_nested, [noop_pl, noop_pl], "nested_in", "nested_out") - p_nested.create_outputs() - p_nested.save() - - # First run - Manager.execute_pipeline(remover, noop_pl, [seq_dataset], groups_allowed=[]) - # Second run - Manager.execute_pipeline(remover, noop_pl, [seq_dataset], groups_allowed=[]) - - two_step_noop_pl = make_first_pipeline( - "Nucleotide Sequence two-step Noop", - "A two-step noop pipeline for nucleotide sequences.", - remover, - grant_everyone_access=False) - create_linear_pipeline(two_step_noop_pl, - [nuc_seq_noop, nuc_seq_noop], - "noop_pipeline_in", - "noop_pipeline_out") - - two_step_seq_datafile = tempfile.NamedTemporaryFile(delete=False) - two_step_seq_datafile.write("""sequence -AAAA -CCCCC -GGGGGG -TTTTTTC - """.encode()) - two_step_seq_datafile.close() - two_step_seq_dataset = Dataset.create_dataset( - file_path=two_step_seq_datafile.name, - user=remover, - cdt=one_col_nuc_seq, - keep_file=True, - name="Removal test data for a two-step Pipeline", - description="A dataset for use in the removal test case with the two-step Pipeline." - ) - - # Two step run - Manager.execute_pipeline( - remover, - two_step_noop_pl, - [two_step_seq_dataset], - groups_allowed=[]) - - -def create_sandbox_testing_tools_environment(case): - case.STR = Datatype.objects.get(pk=datatypes.STR_PK) - - # An ordinary user. - case.user_bob = User.objects.create_user('bob', 'bob@talabs.com', 'verysecure') - case.user_bob.save() - case.user_bob.groups.add(everyone_group()) - case.user_bob.save() - - # Predefined datatypes. - case.datatype_str = new_datatype("my_string", "sequences of ASCII characters", case.STR, case.user_bob) - case.datatype_str.grant_everyone_access() - - # A CDT composed of only one column, strings. - case.cdt_string = CompoundDatatype(user=case.user_bob) - case.cdt_string.save() - case.cdt_string.members.create(datatype=case.datatype_str, column_name="word", column_idx=1) - case.cdt_string.grant_everyone_access() - - # A code resource which does nothing. - case.coderev_noop = make_first_revision( - "noop", "a script to do nothing", "noop.sh", - '#!/bin/bash\n cat "$1" > "$2"', - case.user_bob) - case.coderev_noop.coderesource.grant_everyone_access() - case.coderev_noop.grant_everyone_access() - - # A Method telling Shipyard how to use the noop code on string data. - case.method_noop = make_first_method("string noop", "a method to do nothing to strings", case.coderev_noop, - case.user_bob) - case.method_noop.family.grant_everyone_access() - case.method_noop.grant_everyone_access() - simple_method_io(case.method_noop, case.cdt_string, "strings", "same_strings") - - # Another totally different Method that uses the same CodeRevision and yes it does the same thing. - case.method_trivial = make_first_method( - "string trivial", - "a TOTALLY DIFFERENT method that TOTALLY does SOMETHING to strings by leaving them alone", - case.coderev_noop, - case.user_bob) - case.method_trivial.family.grant_everyone_access() - case.method_trivial.grant_everyone_access() - simple_method_io(case.method_trivial, case.cdt_string, "strings", "untouched_strings") - - # A third one, only this one takes raw input. - case.method_noop_raw = make_first_method("raw noop", "do nothing to raw data", case.coderev_noop, - case.user_bob) - case.method_noop_raw.family.grant_everyone_access() - case.method_noop_raw.grant_everyone_access() - simple_method_io(case.method_noop_raw, None, "raw", "same_raw") - - -def load_sandbox_testing_tools_environment(case): - case.STR = Datatype.objects.get(pk=datatypes.STR_PK) - case.user_bob = User.objects.get(username='bob') - case.datatype_str = Datatype.objects.get(name="my_string") - case.cdt_string = CompoundDatatype.objects.get(members__column_name="word") - case.coderev_noop = CodeResourceRevision.objects.get( - coderesource__name='noop') - - case.method_noop = Method.objects.get(family__name="string noop") - case.method_trivial = Method.objects.get(family__name="string trivial") - case.method_noop_raw = Method.objects.get(family__name="raw noop") - - -def destroy_sandbox_testing_tools_environment(case): - """ - Clean up a TestCase where create_sandbox_testing_tools_environment has been called. - # """ - clean_up_all_files() - - -def create_archive_test_environment(case): - create_librarian_test_environment(case) - create_sandbox_testing_tools_environment(case) - - -def load_archive_test_environment(case): - load_librarian_test_environment(case) - load_sandbox_testing_tools_environment(case) - - -def create_archive_no_runs_test_environment(case): - create_eric_martin_test_environment(case) - create_sandbox_testing_tools_environment(case) - - -def load_archive_no_runs_test_environment(case): - load_eric_martin_test_environment(case) - load_sandbox_testing_tools_environment(case) - - -def create_method_test_environment(case): - """Set up default database state that includes some CRs, CRRs, Methods, etc.""" - # This sets up the DTs and CDTs used in our metadata tests. - create_metadata_test_environment(case) - - fd_count("FD count on environment creation") - - # Define comp_cr - case.comp_cr = CodeResource( - name="complement", - description="Complement DNA/RNA nucleotide sequences", - filename="complement.py", - user=case.myUser) - case.comp_cr.save() - case.comp_cr.grant_everyone_access() - - # Define compv1_crRev for comp_cr - fn = "complement.py" - with open(os.path.join(samplecode_path, fn), "rb") as f: - case.compv1_crRev = CodeResourceRevision( - coderesource=case.comp_cr, - revision_name="v1", - revision_desc="First version", - content_file=File(f), - user=case.myUser) - case.compv1_crRev.full_clean() - case.compv1_crRev.save() - case.compv1_crRev.grant_everyone_access() - - # Define compv2_crRev for comp_cr - fn = "complement_v2.py" - with open(os.path.join(samplecode_path, fn), "rb") as f: - case.compv2_crRev = CodeResourceRevision( - coderesource=case.comp_cr, - revision_name="v2", - revision_desc="Second version: better docstring", - revision_parent=case.compv1_crRev, - content_file=File(f), - user=case.myUser) - case.compv2_crRev.full_clean() - case.compv2_crRev.save() - case.compv2_crRev.grant_everyone_access() - - # Define DNA reference to use as a dependency - dna_resource = CodeResource( - name="dna_ref", - description="Reference DNA sequences", - filename="good_dna.csv", - user=case.myUser) - dna_resource.save() - dna_resource.grant_everyone_access() - fn = "GoodDNANucSeq.csv" - with open(os.path.join(samplecode_path, fn), "rb") as f: - dna_resource_revision = CodeResourceRevision( - coderesource=dna_resource, - revision_name="Prototype", - revision_desc="Reference DNA sequences", - content_file=File(f), - user=case.myUser) - dna_resource_revision.full_clean() - dna_resource_revision.save() - dna_resource_revision.grant_everyone_access() - - # The following is for testing code resource dependencies. - case.test_cr_1 = CodeResource(name="test_cr_1", - filename="test_cr_1.py", - description="CR1", - user=case.myUser) - case.test_cr_1.save() - case.test_cr_1.grant_everyone_access() - case.test_cr_1_rev1 = CodeResourceRevision(coderesource=case.test_cr_1, - revision_name="v1", - revision_desc="CR1-rev1", - user=case.myUser) - - case.test_cr_2 = CodeResource(name="test_cr_2", - filename="test_cr_2.py", - description="CR2", - user=case.myUser) - case.test_cr_2.save() - case.test_cr_2.grant_everyone_access() - case.test_cr_2_rev1 = CodeResourceRevision(coderesource=case.test_cr_2, - revision_name="v1", - revision_desc="CR2-rev1", - user=case.myUser) - - case.test_cr_3 = CodeResource(name="test_cr_3", - filename="test_cr_3.py", - description="CR3", - user=case.myUser) - case.test_cr_3.save() - case.test_cr_3.grant_everyone_access() - case.test_cr_3_rev1 = CodeResourceRevision(coderesource=case.test_cr_3, - revision_name="v1", - revision_desc="CR3-rev1", - user=case.myUser) - case.test_cr_3_rev1.save() - - case.test_cr_4 = CodeResource(name="test_cr_4", - filename="test_cr_4.py", - description="CR4", - user=case.myUser) - case.test_cr_4.save() - case.test_cr_4.grant_everyone_access() - case.test_cr_4_rev1 = CodeResourceRevision(coderesource=case.test_cr_4, - revision_name="v1", - revision_desc="CR4-rev1", - user=case.myUser) - case.test_cr_4_rev1.save() - - fn = "test_cr.py" - with open(os.path.join(samplecode_path, fn), "rb") as f: - md5gen = hashlib.md5() - md5gen.update(f.read()) - f.seek(0) - for crr in [case.test_cr_1_rev1, case.test_cr_2_rev1, case.test_cr_3_rev1, case.test_cr_4_rev1]: - crr.MD5_checksum = md5gen.hexdigest() - crr.content_file.save(fn, File(f)) - - for crr in [case.test_cr_1_rev1, case.test_cr_2_rev1, case.test_cr_3_rev1, case.test_cr_4_rev1]: - crr.save() - crr.grant_everyone_access() - - # Define DNAcomp_mf - case.DNAcomp_mf = MethodFamily( - name="DNAcomplement", - description="Complement DNA nucleotide sequences.", - user=case.myUser) - case.DNAcomp_mf.full_clean() - case.DNAcomp_mf.save() - case.DNAcomp_mf.grant_everyone_access() - - # Define DNAcompv1_m (method revision) for DNAcomp_mf with driver compv1_crRev - case.DNAcompv1_m = case.DNAcomp_mf.members.create( - revision_name="v1", - revision_desc="First version", - driver=case.compv1_crRev, - user=case.myUser) - case.DNAcompv1_m.grant_everyone_access() - - # Add input DNAinput_cdt to DNAcompv1_m - case.DNAinput_ti = case.DNAcompv1_m.create_input( - compounddatatype=case.DNAinput_cdt, - dataset_name="input", - dataset_idx=1) - case.DNAinput_ti.full_clean() - case.DNAinput_ti.save() - - # Add output DNAoutput_cdt to DNAcompv1_m - case.DNAoutput_to = case.DNAcompv1_m.create_output( - compounddatatype=case.DNAoutput_cdt, - dataset_name="output", - dataset_idx=1) - case.DNAoutput_to.full_clean() - case.DNAoutput_to.save() - - # Define DNAcompv2_m for DNAcomp_mf with driver compv2_crRev - # May 20, 2014: where previously the inputs/outputs would be - # automatically copied over from the parent using save(), now - # we explicitly call copy_io_from_parent. - case.DNAcompv2_m = case.DNAcomp_mf.members.create( - revision_name="v2", - revision_desc="Second version", - revision_parent=case.DNAcompv1_m, - driver=case.compv2_crRev, - user=case.myUser) - case.DNAcompv2_m.full_clean() - case.DNAcompv2_m.save() - case.DNAcompv2_m.grant_everyone_access() - case.DNAcompv2_m.copy_io_from_parent() - # case.compv2_crRev requires this to work: - case.DNAcompv2_m.dependencies.create(requirement=dna_resource_revision) - - # Define second family, RNAcomp_mf - case.RNAcomp_mf = MethodFamily( - name="RNAcomplement", - description="Complement RNA nucleotide sequences.", - user=case.myUser) - case.RNAcomp_mf.full_clean() - case.RNAcomp_mf.save() - case.RNAcomp_mf.grant_everyone_access() - - # Define RNAcompv1_m for RNAcomp_mf with driver compv1_crRev - case.RNAcompv1_m = case.RNAcomp_mf.members.create( - revision_name="v1", - revision_desc="First version", - driver=case.compv1_crRev, - user=case.myUser) - case.RNAcompv1_m.grant_everyone_access() - - # Add input RNAinput_cdt to RNAcompv1_m - case.RNAinput_ti = case.RNAcompv1_m.create_input( - compounddatatype=case.RNAinput_cdt, - dataset_name="input", - dataset_idx=1) - case.RNAinput_ti.full_clean() - case.RNAinput_ti.save() - - # Add output RNAoutput_cdt to RNAcompv1_m - case.RNAoutput_to = case.RNAcompv1_m.create_output( - compounddatatype=case.RNAoutput_cdt, - dataset_name="output", - dataset_idx=1) - case.RNAoutput_to.full_clean() - case.RNAoutput_to.save() - - # Define RNAcompv2_m for RNAcompv1_mf with driver compv2_crRev - # May 20, 2014: again, we now explicitly copy over the inputs/outputs. - case.RNAcompv2_m = case.RNAcomp_mf.members.create( - revision_name="v2", - revision_desc="Second version", - revision_parent=case.RNAcompv1_m, - driver=case.compv2_crRev, - user=case.myUser) - case.RNAcompv2_m.full_clean() - case.RNAcompv2_m.save() - case.RNAcompv2_m.copy_io_from_parent() - case.RNAcompv2_m.grant_everyone_access() - # case.compv2_crRev requires this to work: - case.RNAcompv2_m.dependencies.create(requirement=dna_resource_revision) - - # Create method family for script_1_method / script_2_method / script_3_method - case.test_mf = MethodFamily(name="Test method family", - description="Holds scripts 1/2/3", - user=case.myUser) - case.test_mf.full_clean() - case.test_mf.save() - case.test_mf.grant_everyone_access() - - # script_1_sum_and_outputs.py - # INPUT: 1 csv containing (x,y) - # OUTPUT: 1 csv containing (x+y,xy) - case.script_1_cr = CodeResource(name="Sum and product of x and y", - filename="script_1_sum_and_products.py", - description="Addition and multiplication", - user=case.myUser) - case.script_1_cr.save() - case.script_1_cr.grant_everyone_access() - - # Add code resource revision for code resource (script_1_sum_and_products ) - fn = "script_1_sum_and_products.py" - with open(os.path.join(samplecode_path, fn), "rb") as f: - case.script_1_crRev = CodeResourceRevision( - coderesource=case.script_1_cr, - revision_name="v1", - revision_desc="First version", - user=case.myUser, - content_file=File(f) - ) - case.script_1_crRev.full_clean() - case.script_1_crRev.save() - case.script_1_crRev.grant_everyone_access() - - # Establish code resource revision as a method - case.script_1_method = Method( - revision_name="script1", - revision_desc="script1", - family=case.test_mf, - driver=case.script_1_crRev, - user=case.myUser) - case.script_1_method.save() - case.script_1_method.grant_everyone_access() - - # Assign tuple as both an input and an output to script_1_method - case.script_1_method.create_input(compounddatatype=case.tuple_cdt, - dataset_name="input_tuple", - dataset_idx=1) - case.script_1_method.create_output(compounddatatype=case.tuple_cdt, - dataset_name="input_tuple", - dataset_idx=1) - case.script_1_method.full_clean() - case.script_1_method.save() - - # script_2_square_and_means - # INPUT: 1 csv containing (a,b,c) - # OUTPUT-1: 1 csv containing triplet (a^2,b^2,c^2) - # OUTPUT-2: 1 csv containing singlet mean(a,b,c) - case.script_2_cr = CodeResource(name="Square and mean of (a,b,c)", - filename="script_2_square_and_means.py", - description="Square and mean - 2 CSVs", - user=case.myUser) - case.script_2_cr.save() - case.script_2_cr.grant_everyone_access() - - # Add code resource revision for code resource (script_2_square_and_means) - fn = "script_2_square_and_means.py" - with open(os.path.join(samplecode_path, fn), "rb") as f: - case.script_2_crRev = CodeResourceRevision( - coderesource=case.script_2_cr, - revision_name="v1", - revision_desc="First version", - user=case.myUser, - content_file=File(f)) - case.script_2_crRev.full_clean() - case.script_2_crRev.save() - case.script_2_crRev.grant_everyone_access() - - # Establish code resource revision as a method - case.script_2_method = Method( - revision_name="script2", - revision_desc="script2", - family=case.test_mf, - driver=case.script_2_crRev, - user=case.myUser) - case.script_2_method.save() - case.script_2_method.grant_everyone_access() - - # Assign triplet as input and output, - case.script_2_method.create_input( - compounddatatype=case.triplet_cdt, - dataset_name="a_b_c", - dataset_idx=1) - case.script_2_method.create_output( - compounddatatype=case.triplet_cdt, - dataset_name="a_b_c_squared", - dataset_idx=1) - case.script_2_method.create_output( - compounddatatype=case.singlet_cdt, - dataset_name="a_b_c_mean", - dataset_idx=2) - case.script_2_method.full_clean() - case.script_2_method.save() - - # script_3_product - # INPUT-1: Single column (k) - # INPUT-2: Single-row, single column (r) - # OUTPUT-1: Single column r*(k) - case.script_3_cr = CodeResource(name="Scalar multiple of k", - filename="script_3_product.py", - description="Product of input", - user=case.myUser) - case.script_3_cr.save() - case.script_3_cr.grant_everyone_access() - - # Add code resource revision for code resource (script_3_product) - with open(os.path.join(samplecode_path, "script_3_product.py"), "rb") as f: - case.script_3_crRev = CodeResourceRevision( - coderesource=case.script_3_cr, - revision_name="v1", - revision_desc="First version", - content_file=File(f), - user=case.myUser) - case.script_3_crRev.full_clean() - case.script_3_crRev.save() - case.script_3_crRev.grant_everyone_access() - - # Establish code resource revision as a method - case.script_3_method = Method( - revision_name="script3", - revision_desc="script3", - family=case.test_mf, - driver=case.script_3_crRev, - user=case.myUser) - case.script_3_method.save() - case.script_3_method.grant_everyone_access() - - # Assign singlet as input and output - case.script_3_method.create_input(compounddatatype=case.singlet_cdt, - dataset_name="k", - dataset_idx=1) - - case.script_3_method.create_input(compounddatatype=case.singlet_cdt, - dataset_name="r", - dataset_idx=2, - max_row=1, - min_row=1) - - case.script_3_method.create_output(compounddatatype=case.singlet_cdt, - dataset_name="kr", - dataset_idx=1) - case.script_3_method.full_clean() - case.script_3_method.save() - - #### - # This next bit was originally in pipeline.tests. - - # DNArecomp_mf is a MethodFamily called DNArecomplement - case.DNArecomp_mf = MethodFamily( - name="DNArecomplement", - description="Re-complement DNA nucleotide sequences.", - user=case.myUser) - case.DNArecomp_mf.full_clean() - case.DNArecomp_mf.save() - case.DNArecomp_mf.grant_everyone_access() - - # Add to MethodFamily DNArecomp_mf a method revision DNArecomp_m - case.DNArecomp_m = case.DNArecomp_mf.members.create( - revision_name="v1", - revision_desc="First version", - driver=case.compv2_crRev, - user=case.myUser) - case.DNArecomp_m.grant_everyone_access() - # case.compv2_crRev requires this to work: - case.DNArecomp_m.dependencies.create(requirement=dna_resource_revision) - - # To this method revision, add inputs with CDT DNAoutput_cdt - case.DNArecomp_m.create_input( - compounddatatype=case.DNAoutput_cdt, - dataset_name="complemented_seqs", - dataset_idx=1) - - # To this method revision, add outputs with CDT DNAinput_cdt - case.DNArecomp_m.create_output( - compounddatatype=case.DNAinput_cdt, - dataset_name="recomplemented_seqs", - dataset_idx=1) - - # Setup used in the "2nd-wave" tests (this was originally in - # Copperfish_Raw_Setup). - - # Define CR "script_4_raw_in_CSV_out.py" - # input: raw [but contains (a,b,c) triplet] - # output: CSV [3 CDT members of the form (a^2, b^2, c^2)] - - # Define CR in order to define CRR - case.script_4_CR = CodeResource( - name="Generate (a^2, b^2, c^2) using RAW input", - filename="script_4_raw_in_CSV_out.py", - description="Given (a,b,c), outputs (a^2,b^2,c^2)", - user=case.myUser) - case.script_4_CR.save() - case.script_4_CR.grant_everyone_access() - - # Define CRR for this CR in order to define method - with open(os.path.join(samplecode_path, "script_4_raw_in_CSV_out.py"), "rb") as f: - case.script_4_1_CRR = CodeResourceRevision( - coderesource=case.script_4_CR, - revision_name="v1", - revision_desc="v1", - content_file=File(f), - user=case.myUser) - case.script_4_1_CRR.full_clean() - case.script_4_1_CRR.save() - case.script_4_1_CRR.grant_everyone_access() - - # Define MF in order to define method - case.test_MF = MethodFamily( - name="test method family", - description="method family placeholder", - user=case.myUser) - case.test_MF.full_clean() - case.test_MF.save() - case.test_MF.grant_everyone_access() - - # Establish CRR as a method within a given method family - case.script_4_1_M = Method( - revision_name="s4", - revision_desc="s4", - family=case.test_MF, - driver=case.script_4_1_CRR, - user=case.myUser) - case.script_4_1_M.save() - case.script_4_1_M.grant_everyone_access() - - case.script_4_1_M.create_input(compounddatatype=case.triplet_cdt, - dataset_name="s4_input", - dataset_idx=1) - case.script_4_1_M.full_clean() - - # A shorter alias - case.testmethod = case.script_4_1_M - - # Some code for a no-op method. - resource = CodeResource(name="noop", filename="noop.sh", user=case.myUser) - resource.save() - resource.grant_everyone_access() - file_prefix = os.path.join(settings.MEDIA_ROOT, - CodeResourceRevision.UPLOAD_DIR, - 'noop') - with tempfile.NamedTemporaryFile(prefix=file_prefix, suffix='.sh') as f: - f.write("#!/bin/bash\ncat $1".encode()) - case.noop_data_file = f.name - revision = CodeResourceRevision(coderesource=resource, - content_file=File(f), - user=case.myUser) - revision.clean() - revision.save() - revision.grant_everyone_access() - - # Retrieve the string type. - string_dt = Datatype.objects.get(pk=datatypes.STR_PK) - string_cdt = CompoundDatatype(user=case.myUser) - string_cdt.save() - string_cdt.members.create(datatype=string_dt, column_name="word", column_idx=1) - string_cdt.grant_everyone_access() - string_cdt.full_clean() - - mfamily = MethodFamily(name="noop", user=case.myUser) - mfamily.save() - mfamily.grant_everyone_access() - case.noop_method = Method(family=mfamily, - driver=revision, - revision_name="1", - revision_desc="first version", - user=case.myUser) - case.noop_method.save() - case.noop_method.create_input(compounddatatype=string_cdt, - dataset_name="noop_data", - dataset_idx=1) - case.noop_method.grant_everyone_access() - case.noop_method.full_clean() - - # Some data. - case.scratch_dir = tempfile.mkdtemp( - dir=file_access_utils.create_sandbox_base_path() - ) - file_access_utils.configure_sandbox_permissions(case.scratch_dir) - try: - fd, case.noop_infile = tempfile.mkstemp(dir=case.scratch_dir) - finally: - os.close(fd) - try: - fd, case.noop_outfile = tempfile.mkstemp(dir=case.scratch_dir) - finally: - os.close(fd) - case.noop_indata = "word\nhello\nworld" - - with open(case.noop_infile, "wt") as handle: - handle.write(case.noop_indata) - - file_access_utils.configure_sandbox_permissions(case.noop_infile) - file_access_utils.configure_sandbox_permissions(case.noop_outfile) - - -def destroy_method_test_environment(case): - """ - Clean up a TestCase where create_method_test_environment has been called. - """ - clean_up_all_files() - shutil.rmtree(case.scratch_dir) - CodeResource.objects.all().delete() - - -def create_pipeline_test_environment(case): - """ - Sets up default database state for some Pipeline unit testing. - - This also sets up Methods, CR/CRR/CRDs, and DTs/CDTs as in the Metadata and Methods tests. - """ - create_method_test_environment(case) - case.workdir = tempfile.mkdtemp() - - case.user = User.objects.create_user('bob', 'bob@aol.com', '12345') - case.user.save() - - # Define DNAcomp_pf - case.DNAcomp_pf = PipelineFamily(name="DNAcomplement", description="DNA complement pipeline.", - user=case.user) - case.DNAcomp_pf.save() - - # Define DNAcompv1_p (pipeline revision) - case.DNAcompv1_p = case.DNAcomp_pf.members.create(revision_name="v1", revision_desc="First version", - user=case.user) - - # Add Pipeline input CDT DNAinput_cdt to pipeline revision DNAcompv1_p - case.DNAcompv1_p.create_input( - compounddatatype=case.DNAinput_cdt, - dataset_name="seqs_to_complement", - dataset_idx=1) - - # Add a step to Pipeline revision DNAcompv1_p involving - # a transformation DNAcompv2_m at step 1 - step1 = case.DNAcompv1_p.steps.create( - transformation=case.DNAcompv2_m, - step_num=1) - - # Add cabling (PipelineStepInputCable's) to (step1, DNAcompv1_p) - # From step 0, output hole "seqs_to_complement" to - # input hole "input" (of this step) - step1.cables_in.create(dest=case.DNAcompv2_m.inputs.get(dataset_name="input"), source_step=0, - source=case.DNAcompv1_p.inputs.get(dataset_name="seqs_to_complement")) - - # Add output cabling (PipelineOutputCable) to DNAcompv1_p - # From step 1, output hole "output", send output to - # Pipeline output hole "complemented_seqs" at index 1 - case.DNAcompv1_p.create_outcable(source_step=1, - source=step1.transformation.outputs.get(dataset_name="output"), - output_name="complemented_seqs", output_idx=1) - - # Define PF in order to define pipeline - case.test_PF = PipelineFamily( - name="test pipeline family", - description="pipeline family placeholder", - user=case.user) - case.test_PF.full_clean() - case.test_PF.save() - - # Set up an empty Pipeline. - family = PipelineFamily.filter_by_user(case.user).first() - - # Nothing defined. - p = Pipeline(family=family, revision_name="foo", revision_desc="Foo version", user=case.user) - p.save() - - -def destroy_pipeline_test_environment(case): - """ - Clean up a TestCase where create_pipeline_test_environment has been called. - """ - destroy_method_test_environment(case) - Dataset.objects.all().delete() - shutil.rmtree(case.workdir) - - -def create_sequence_manipulation_environment(case): - create_sandbox_testing_tools_environment(case) - - # Alice is a Shipyard user. - case.user_alice = User.objects.create_user('alice', 'alice@talabs.com', 'secure') - case.user_alice.save() - case.user_alice.groups.add(everyone_group()) - case.user_alice.save() - - # Alice's lab has two tasks - complement DNA, and reverse and complement DNA. - # She wants to create a pipeline for each. In the background, this also creates - # two new pipeline families. - case.pipeline_complement = make_first_pipeline("DNA complement", "a pipeline to complement DNA", case.user_alice) - case.pipeline_reverse = make_first_pipeline("DNA reverse", "a pipeline to reverse DNA", case.user_alice) - case.pipeline_revcomp = make_first_pipeline("DNA revcomp", "a pipeline to reverse and complement DNA", - case.user_alice) - - # Alice is only going to be manipulating DNA, so she creates a "DNA" - # data type. A "string" datatype, which she will use for the headers, - # has been predefined in Shipyard. She also creates a compound "record" - # datatype for sequence + header. - case.datatype_dna = new_datatype("DNA", "sequences of ATCG", case.STR, case.user_alice) - case.cdt_record = CompoundDatatype(user=case.user_alice) - case.cdt_record.save() - case.cdt_record.members.create(datatype=case.datatype_str, column_name="header", column_idx=1) - case.cdt_record.members.create(datatype=case.datatype_dna, column_name="sequence", column_idx=2) - case.cdt_record.grant_everyone_access() - - # Alice uploads code to perform each of the tasks. In the background, - # Shipyard creates new CodeResources for these scripts and sets her - # uploaded files as the first CodeResourceRevisions. - case.coderev_complement = make_first_revision( - "DNA complement", "a script to complement DNA", - "complement.sh", - """#!/bin/bash - cat "$1" | cut -d ',' -f 2 | tr 'ATCG' 'TAGC' | paste -d, "$1" - | cut -d ',' -f 1,3 > "$2" - """, - case.user_alice) - case.coderev_reverse = make_first_revision( - "DNA reverse", "a script to reverse DNA", "reverse.sh", - """#!/bin/bash - cat "$1" | cut -d ',' -f 2 | rev | paste -d, "$1" - | cut -d ',' -f 1,3 > "$2" - """, - case.user_alice) - - # To tell the system how to use her code, Alice creates two Methods, - # one for each CodeResource. In the background, this creates two new - # MethodFamilies with her Methods as the first member of each. - case.method_complement = make_first_method("DNA complement", "a method to complement strings of DNA", - case.coderev_complement, - case.user_alice) - simple_method_io(case.method_complement, case.cdt_record, "DNA_to_complement", "complemented_DNA") - case.method_reverse = make_first_method("DNA reverse", "a method to reverse strings of DNA", - case.coderev_complement, - case.user_alice) - simple_method_io(case.method_reverse, case.cdt_record, "DNA_to_reverse", "reversed_DNA") - - # Now Alice is ready to define her pipelines. She uses the GUI to drag - # the "complement" method into the "complement" pipeline, creates - # the pipeline's input and output, and connects them to the inputs and - # output of the method. - create_linear_pipeline(case.pipeline_complement, [case.method_complement], "lab_data", - "complemented_lab_data") - case.pipeline_complement.create_outputs() - create_linear_pipeline(case.pipeline_reverse, [case.method_reverse], "lab_data", "reversed_lab_data") - case.pipeline_reverse.create_outputs() - create_linear_pipeline(case.pipeline_revcomp, [case.method_reverse, case.method_complement], "lab_data", - "reverse_and_complemented_lab_data") - case.pipeline_revcomp.create_outputs() - - # Here is some data which is sitting on Alice's hard drive. - random.seed("Constant seed avoids intermittent failures.") - case.labdata = "header,sequence\n" - for i in range(10): - seq = "".join([random.choice("ATCG") for _ in range(10)]) - case.labdata += "patient{},{}\n".format(i, seq) - case.datafile = tempfile.NamedTemporaryFile( - delete=False, - dir=file_access_utils.sandbox_base_path() - ) - case.datafile.write(case.labdata.encode()) - case.datafile.close() - file_access_utils.configure_sandbox_permissions(case.datafile.name) - - # Alice uploads the data to the system. - case.dataset_labdata = Dataset.create_dataset( - file_path=case.datafile.name, - user=case.user_alice, - cdt=case.cdt_record, - keep_file=True, - name="lab data", - description="data from the lab" - ) - - # A second version of the complement Pipeline which doesn't keep any output. - case.pipeline_complement_v2 = Pipeline(family=case.pipeline_complement.family, revision_name="2", - revision_desc="second version", user=case.user_alice) - case.pipeline_complement_v2.save() - create_linear_pipeline(case.pipeline_complement_v2, - [case.method_complement], - "lab_data", - "complemented_lab_data") - case.pipeline_complement_v2.steps.last().add_deletion(case.method_complement.outputs.first()) - case.pipeline_complement_v2.outcables.first().delete() - case.pipeline_complement_v2.create_outputs() - - # A second version of the reverse/complement Pipeline which doesn't keep - # intermediate or final output. - case.pipeline_revcomp_v2 = Pipeline(family=case.pipeline_revcomp.family, revision_name="2", - revision_desc="second version", user=case.user_alice) - case.pipeline_revcomp_v2.save() - create_linear_pipeline(case.pipeline_revcomp_v2, - [case.method_reverse, case.method_complement], - "lab_data", - "revcomped_lab_data") - case.pipeline_revcomp_v2.steps.get(step_num=1).add_deletion(case.method_reverse.outputs.first()) - case.pipeline_revcomp_v2.steps.get(step_num=2).add_deletion(case.method_complement.outputs.first()) - case.pipeline_revcomp_v2.outcables.first().delete() - case.pipeline_revcomp_v2.create_outputs() - - # A third version of the reverse/complement Pipeline which keeps - # final output, but not intermediate. - case.pipeline_revcomp_v3 = Pipeline(family=case.pipeline_revcomp.family, revision_name="3", - revision_desc="third version", user=case.user_alice) - case.pipeline_revcomp_v3.save() - create_linear_pipeline(case.pipeline_revcomp_v3, [case.method_reverse, case.method_complement], - "lab_data", "revcomped_lab_data") - case.pipeline_revcomp_v3.steps.get(step_num=1).add_deletion(case.method_reverse.outputs.first()) - case.pipeline_revcomp_v3.create_outputs() - - # Another method which turns DNA into RNA. - source = """\ -#!/bin/bash -cat "$1" | cut -d ',' -f 2 | tr 'T' 'U' | paste -d, "$1" - | cut -d ',' -f 1,3 > "$2" -""" - case.coderev_DNA2RNA = make_first_revision("DNA to RNA", - "a script to reverse DNA", - "DNA2RNA.sh", - source, - case.user_alice) - case.method_DNA2RNA = make_first_method("DNA to RNA", "a method to turn strings of DNA into RNA", - case.coderev_DNA2RNA, case.user_alice) - simple_method_io(case.method_DNA2RNA, case.cdt_record, "DNA_to_convert", "RNA") - - # A pipeline which reverses DNA, then turns it into RNA. - case.pipeline_revRNA = make_first_pipeline( - "DNA to reversed RNA", - "a pipeline to reverse DNA and translate it to RNA", - case.user_alice) - create_linear_pipeline(case.pipeline_revRNA, [case.method_reverse, case.method_DNA2RNA], "lab_data", - "RNAd_lab_data") - case.pipeline_revRNA.create_outputs() - - # Separator to print between Pipeline executions, to make viewing logs easier. - case.sep = " "*80 + "\n" + "*"*80 + "\n" + " "*80 + "\n" - - # Figure out the MD5 of the output file created when the complement method - # is run on Alice's data, so we can check it later. - tmpdir = tempfile.mkdtemp(dir=file_access_utils.sandbox_base_path()) - file_access_utils.configure_sandbox_permissions(tmpdir) - - outfile = os.path.join(tmpdir, "output") - stdout_path = os.path.join(tmpdir, "stdout.txt") - stderr_path = os.path.join(tmpdir, "stderr.txt") - - # Run this code using the DummySlurmScheduler; it returns a SlurmJobHandle - # and the job should be done. - case.method_complement.install(tmpdir) - complement_sjh = case.method_complement.submit_code( - tmpdir, - [case.datafile.name], - [outfile], - stdout_path, - stderr_path, - slurm_sched_class=DummySlurmScheduler - ) - while complement_sjh.get_state() != DummySlurmScheduler.COMPLETED: - time.sleep(1) - - case.labdata_compd_md5 = file_access_utils.compute_md5(open(outfile)) - shutil.rmtree(tmpdir) - - -def destroy_sequence_manipulation_environment(case): - clean_up_all_files() - if os.path.exists(case.datafile.name): - os.remove(case.datafile.name) - - -def create_word_reversal_environment(case): - """ - Create an environment with some word-reversal code and pipelines. - """ - create_sandbox_testing_tools_environment(case) - - # A code resource which reverses a file. - case.coderev_reverse = make_first_revision( - "reverse", - "a script to reverse lines of a file", - "reverse.py", - ("#!/usr/bin/env python\n" - "import sys\n" - "import csv\n" - "with open(sys.argv[1]) as infile, open(sys.argv[2], 'w') as outfile:\n" - " reader = csv.reader(infile)\n" - " writer = csv.writer(outfile)\n" - " for row in reader:\n" - " writer.writerow([row[1][::-1], row[0][::-1]])\n"), - case.user_bob) - - # A CDT with two columns, word and drow. - case.cdt_wordbacks = CompoundDatatype(user=case.user_bob) - case.cdt_wordbacks.save() - case.cdt_wordbacks.members.create(datatype=case.datatype_str, column_name="word", column_idx=1) - case.cdt_wordbacks.members.create(datatype=case.datatype_str, column_name="drow", column_idx=2) - case.cdt_wordbacks.grant_everyone_access() - - # A second CDT, much like the first :] - case.cdt_backwords = CompoundDatatype(user=case.user_bob) - case.cdt_backwords.save() - case.cdt_backwords.members.create(datatype=case.datatype_str, column_name="drow", column_idx=1) - case.cdt_backwords.members.create(datatype=case.datatype_str, column_name="word", column_idx=2) - case.cdt_backwords.grant_everyone_access() - - # Methods for the reverse CRR, and noop CRR. - case.method_reverse = make_first_method("string reverse", "a method to reverse strings", - case.coderev_reverse, case.user_bob) - simple_method_io(case.method_reverse, case.cdt_wordbacks, "words_to_reverse", "reversed_words") - case.method_re_reverse = make_first_method("string re-reverse", "a method to re-reverse strings", - case.coderev_reverse, case.user_bob) - simple_method_io(case.method_re_reverse, case.cdt_backwords, "words_to_rereverse", "rereversed_words") - - case.method_noop_wordbacks = make_first_method( - "noop wordback", - "a method to do nothing on two columns (word, drow)", - case.coderev_noop, - case.user_bob) - simple_method_io(case.method_noop_wordbacks, case.cdt_wordbacks, "words", "more_words") - case.method_noop_backwords = make_first_method( - "noop backword", - "a method to do nothing on two columns", - case.coderev_noop, - case.user_bob) - simple_method_io(case.method_noop_backwords, case.cdt_backwords, "backwords", "more_backwords") - - # Some data of type (case.datatype_str: word). - string_datafile = tempfile.NamedTemporaryFile(delete=False) - string_datafile.write("word\n".encode()) - string_datafile.close() - os.system("head -1 /usr/share/dict/words >> {}". - format(string_datafile.name)) - case.dataset_words = Dataset.create_dataset( - file_path=string_datafile.name, - user=case.user_bob, - groups_allowed=[everyone_group()], - cdt=case.cdt_string, - keep_file=True, - name="blahblah", - description="blahblahblah" - ) - - os.remove(string_datafile.name) - - # Some data of type (case.datatype_str: word, case.datatype_str: drow). - case.wordbacks_datafile = tempfile.NamedTemporaryFile(delete=False) - writer = csv.writer(case.wordbacks_datafile) - writer.writerow(["word", "drow"]) - random.seed("Constant seed avoids intermittent failures.") - for _ in range(20): - i = random.randint(1, 99171) - sed = subprocess.Popen(["sed", "{}q;d".format(i), "/usr/share/dict/words"], - stdout=subprocess.PIPE) - word, _ = sed.communicate() - word = word.strip() - writer.writerow([word, word[::-1]]) - case.wordbacks_datafile.close() - - case.backwords_datafile = tempfile.NamedTemporaryFile(delete=False) - writer = csv.writer(case.backwords_datafile) - writer.writerow(["drow", "word"]) - for _ in range(20): - i = random.randint(1, 99171) - sed = subprocess.Popen(["sed", "{}q;d".format(i), "/usr/share/dict/words"], - stdout=subprocess.PIPE) - word, _ = sed.communicate() - word = word.strip() - writer.writerow([word[::-1], word]) - case.backwords_datafile.close() - - case.dataset_wordbacks = Dataset.create_dataset( - file_path=case.wordbacks_datafile.name, - user=case.user_bob, - groups_allowed=[everyone_group()], - cdt=case.cdt_wordbacks, - keep_file=True, - name="wordbacks", - description="random reversed words" - ) - - case.dataset_backwords = Dataset.create_dataset( - file_path=case.backwords_datafile.name, - user=case.user_bob, - groups_allowed=[everyone_group()], - cdt=case.cdt_backwords, - keep_file=True, - name="backwords", - description="random reversed words" - ) - - -def destroy_word_reversal_environment(case): - clean_up_all_files() - if hasattr(case, "words_datafile"): - os.remove(case.words_datafile.name) - - -def make_crisscross_cable(cable): - """ - Helper to take a cable whose source and destination CDTs both have two columns that can be - reversed (e.g. string-string or int-int, etc.) and add "crisscross" wiring. - """ - source_cdt = cable.source.structure.compounddatatype - dest_cdt = cable.dest.structure.compounddatatype - cable.custom_wires.create(source_pin=source_cdt.members.get(column_idx=1), - dest_pin=dest_cdt.members.get(column_idx=2)) - cable.custom_wires.create(source_pin=source_cdt.members.get(column_idx=2), - dest_pin=dest_cdt.members.get(column_idx=1)) - - -def new_datatype(dtname, dtdesc, kivetype, user, grant_everyone_access=True): - """ - Helper function to create a new datatype. - """ - datatype = Datatype(name=dtname, description=dtdesc, user=user) - datatype.save() - datatype.restricts.add(Datatype.objects.get(pk=kivetype.pk)) - if grant_everyone_access: - datatype.grant_everyone_access() - # datatype.complete_clean() - return datatype - - -def make_first_revision(resname, resdesc, resfn, contents, user, grant_everyone_access=True): - """ - Helper function to make a CodeResource and the first version. - """ - assert isinstance(resname, six.string_types), "string expected for resname" - assert isinstance(resdesc, six.string_types), "string expected for resdesc" - assert isinstance(resfn, six.string_types), "string expected for resfn" - assert isinstance(contents, six.string_types), "string expected for contents" - resource = CodeResource(name=resname, description=resdesc, filename=resfn, user=user) - # resource.clean() - resource.save() - if grant_everyone_access: - resource.grant_everyone_access() - cont_bytes = contents.encode() - with tempfile.TemporaryFile() as f: - f.write(cont_bytes) - with transaction.atomic(): - # NOTE: 2018-05-30: must provide a name to File for py3. - revision = CodeResourceRevision( - coderesource=resource, - revision_name="1", - revision_desc="first version", - content_file=File(f, name="blaname"), - user=user) - - # We need to set the MD5. - md5gen = hashlib.md5() - md5gen.update(cont_bytes) - revision.MD5_checksum = md5gen.hexdigest() - revision.save() - revision.clean() - if grant_everyone_access: - revision.grant_everyone_access() - resource.clean() - return revision - - -def make_first_method(famname, famdesc, driver, user, grant_everyone_access=True): - """ - Helper function to make a new MethodFamily for a new Method. - """ - family = MethodFamily(name=famname, description=famdesc, user=user) - family.save() - if grant_everyone_access: - family.grant_everyone_access() - with transaction.atomic(): - method = Method( - revision_name="v1", - revision_desc="first version", - family=family, - driver=driver, - user=user) - method.save() - method.clean() - if grant_everyone_access: - method.grant_everyone_access() - family.clean() - return method - - -def simple_method_io(method, cdt, indataname, outdataname): - """ - Helper function to create inputs and outputs for a simple - Method with one input, one output, and the same CompoundDatatype - for both incoming and outgoing data. - """ - minput = method.create_input(compounddatatype=cdt, - dataset_name=indataname, - dataset_idx=1) - minput.clean() - moutput = method.create_output(compounddatatype=cdt, - dataset_name=outdataname, - dataset_idx=1) - moutput.clean() - method.clean() - return minput, moutput - - -def make_first_pipeline(pname, pdesc, user, grant_everyone_access=True): - """ - Helper function to make a new PipelineFamily and the first Pipeline - member. - """ - family = PipelineFamily(name=pname, description=pdesc, user=user) - family.save() - if grant_everyone_access: - family.grant_everyone_access() - pipeline = Pipeline(family=family, revision_name="v1", revision_desc="first version", user=user) - pipeline.clean() - pipeline.save() - if grant_everyone_access: - pipeline.grant_everyone_access() - family.clean() - return pipeline - - -def make_second_pipeline(pipeline, grant_everyone_access=True): - """ - Create a second version of a Pipeline, in the same family as the first, - without making any changes. Hook up the steps to each other, but don't - create inputs and outputs for the new Pipeline. - """ - new_pipeline = Pipeline(family=pipeline.family, revision_name="v2", revision_desc="second version", - user=pipeline.user) - new_pipeline.save() - if grant_everyone_access: - new_pipeline.grant_everyone_access() - - for step in pipeline.steps.all(): - new_step = new_pipeline.steps.create(transformation=step.transformation, step_num=step.step_num) - for cable in step.cables_in.all(): - if cable.source.transformation.__class__.__name__ == "PipelineStep": - new_step.cables_in.create(source=cable.source, dest=cable.dest) - return new_pipeline - - -def create_linear_pipeline(pipeline, methods, indata, outdata): - """ - Helper function to create a "linear" pipeline, ie. - - ___ __ - in --| |-...-| |-- out - |___| |__| - - indata and outdata are the names of the input and output datasets. - """ - # Create pipeline input. - if methods[0].inputs.first().is_raw(): - cdt_in = None - else: - cdt_in = methods[0].inputs.first().structure.compounddatatype - pipeline_in = pipeline.create_input(compounddatatype=cdt_in, dataset_name=indata, dataset_idx=1) - - # Create steps. - steps = [] - for i, _method in enumerate(methods): - step = pipeline.steps.create(transformation=methods[i], step_num=i+1) - if i == 0: - source = pipeline_in - else: - source = methods[i-1].outputs.first() - step.cables_in.create(source_step=i, source=source, dest=methods[i].inputs.first()) - step.complete_clean() - steps.append(step) - - # Create pipeline output. - pipeline.create_outcable(output_name=outdata, output_idx=1, source_step=len(steps), - source=methods[-1].outputs.first()) - pipeline.create_outputs() - pipeline.complete_clean() - - -# This is potentially slow so we don't just build it into the create_... function above. -# This is also kind of a hack -- depends on case.user_bob and case.cdt_string being present. -def make_words_dataset(case): - """ - Set up a data file of words in the specified test case. - - PRE: the specified test case has a member CDT called cdt_string and user user_bob. - """ - string_datafile = tempfile.NamedTemporaryFile(delete=False) - string_datafile.write("word\n".encode()) - string_datafile.close() - os.system("head -1 /usr/share/dict/words >> {}". - format(string_datafile.name)) - case.dataset_words = Dataset.create_dataset( - file_path=string_datafile.name, - user=case.user_bob, - cdt=case.cdt_string, - keep_file=True, - name="blahblah", - description="blahblahblah" - ) - case.dataset_words.grant_everyone_access() - case.dataset_words.save() - - os.remove(string_datafile.name) - - -# An environment resulting from a user that's messed things up. -def create_grandpa_sandbox_environment(case): - create_sandbox_testing_tools_environment(case) - - # A guy who doesn't know what he is doing. - # May 14, 2014: dag, yo -- RL - # May 20, 2014: he's doing his best, man -- RL - case.user_grandpa = User.objects.create_user('grandpa', 'gr@nd.pa', '123456') - case.user_grandpa.save() - case.user_grandpa.groups.add(everyone_group()) - case.user_grandpa.save() - - # A code resource, method, and pipeline which are empty. - case.coderev_faulty = make_first_revision( - "faulty", - "a script...?", - "faulty.sh", - "", - case.user_grandpa - ) - case.method_faulty = make_first_method( - "faulty", - "a method to... uh...", - case.coderev_faulty, - case.user_grandpa - ) - case.method_faulty.clean() - simple_method_io(case.method_faulty, case.cdt_string, "strings", "i_dont_know") - case.pipeline_faulty = make_first_pipeline("faulty pipeline", "a pipeline to do nothing", case.user_grandpa) - create_linear_pipeline(case.pipeline_faulty, [case.method_faulty, case.method_noop], "data", "the_abyss") - case.pipeline_faulty.create_outputs() - - # A code resource, method, and pipeline which fail. - case.coderev_fubar = make_first_revision( - "fubar", "a script which always fails", - "fubar.sh", "#!/bin/bash\nexit 1", - case.user_grandpa - ) - case.method_fubar = make_first_method("fubar", "a method which always fails", case.coderev_fubar, - case.user_grandpa) - case.method_fubar.clean() - simple_method_io(case.method_fubar, case.cdt_string, "strings", "broken_strings") - case.pipeline_fubar = make_first_pipeline("fubar pipeline", "a pipeline which always fails", case.user_grandpa) - create_linear_pipeline(case.pipeline_fubar, - [case.method_noop, case.method_fubar, case.method_noop], "indata", "outdata") - case.pipeline_fubar.create_outputs() - - # Some data to run through the faulty pipelines. - case.grandpa_datafile = tempfile.NamedTemporaryFile(delete=False) - case.grandpa_datafile.write("word\n".encode()) - random.seed("Constant seed avoids intermittent failures.") - for _ in range(20): - i = random.randint(1, 99171) - case.grandpa_datafile.write("{}\n".format(i).encode()) - case.grandpa_datafile.close() - case.dataset_grandpa = Dataset.create_dataset( - file_path=case.grandpa_datafile.name, - user=case.user_grandpa, - cdt=case.cdt_string, - keep_file=True, - name="numbers", - description="numbers which are actually strings" - ) - case.dataset_grandpa.clean() - - -def destroy_grandpa_sandbox_environment(case): - clean_up_all_files() - os.remove(case.grandpa_datafile.name) - - -def make_dataset(contents, CDT, keep_file, user, name, description, file_source, check): - """ - Wrapper for create_dataset that creates a Dataset from a string. - """ - with tempfile.NamedTemporaryFile(mode="wb", delete=False) as f: - f.write(contents.encode()) - fname = f.name - with open(fname, "r") as f: - test_dataset = Dataset.create_dataset( - None, - user, - cdt=CDT, - keep_file=keep_file, - name=name, - description=description, - file_source=file_source, - check=check, - file_handle=f - ) - - return test_dataset diff --git a/kive/librarian/management/commands/copy_and_check_md5s.py b/kive/librarian/management/commands/copy_and_check_md5s.py deleted file mode 100644 index d5550ec70..000000000 --- a/kive/librarian/management/commands/copy_and_check_md5s.py +++ /dev/null @@ -1,207 +0,0 @@ -""" Copy datasets and check MD5's. Makes no changes to the database. """ -import random -import logging -import os -import shutil -import sys -from time import sleep -from traceback import format_exc - -from django.core.management.base import BaseCommand -from mpi4py import MPI - -from file_access_utils import sandbox_base_path, compute_md5 -from fleet.workers import adjust_log_files -from librarian.models import Dataset - -logger = logging.getLogger('copy_and_check_md5s') - - -class Command(BaseCommand): - help = "Trying to reproduce issue #550 by copying datasets and checking MD5's." - - def add_arguments(self, parser): - parser.add_argument("-n", - "--count", - type=int, - help="number of workers to launch", - default=0) - parser.add_argument("-w", "--worker", action="store_true", help="run as a worker") - parser.add_argument("-t", - "--target", - type=int, - help="target load, or number of simultaneous copies", - default=1) - parser.add_argument("-l", "--limit", type=int, help="limit of files in each sandbox", default=10) - parser.add_argument("-q", - "--query", - type=int, - help="datasets to query, negative for no limit", - default=-1) - parser.add_argument("-p", "--pattern", help="file name pattern in datasets") - - def handle(self, *args, **options): - worker_count = options["count"] - if worker_count: - adjust_log_files(logger, 999) - logger.info('Launching %d workers.', worker_count) - comm = self.launch_workers(worker_count, options) - logger.info('Launched %d workers.', worker_count) - elif options["worker"]: - self.run_worker(options["limit"]) - return - else: - raise RuntimeError('Did not specify either worker or count options.') - - try: - idle_ranks = [] - while len(idle_ranks) < worker_count: - self.receive_and_log(comm, idle_ranks) - - target_load = options["target"] - for dataset_id in self.find_datasets(options["query"]): - while len(idle_ranks) - 1 < worker_count - target_load: - self.receive_and_log(comm, idle_ranks) - rank = idle_ranks.pop(random.randrange(len(idle_ranks))) - comm.send(dataset_id, dest=rank) - except KeyboardInterrupt: - pass - logger.info('Stopping.') - - def receive_and_log(self, comm, idle_ranks): - """ Wait for a message, then log it and record the source rank. - - :param comm: MPI communicator - :param list idle_ranks: will have the source rank added to it - """ - - status = MPI.Status() - logger.debug('Manager waiting to receive.') - level, message = self.polling_receive(comm, status) - logger.log(level, message) - idle_ranks.append(status.source) - - def polling_receive(self, comm, status=None, source=MPI.ANY_SOURCE): - # Set this to 0 for maximum responsiveness, but that will peg CPU to 100% - sleep_seconds = 0.1 - if sleep_seconds > 0: - while not comm.Iprobe(source=source): - sleep(sleep_seconds) - - return comm.recv(source=source, status=status) - - def create_sandbox(self, rank): - sandbox_path = os.path.join(sandbox_base_path(), - 'copy_and_check_md5s_{}'.format(rank)) - shutil.rmtree(sandbox_path, ignore_errors=True) - os.makedirs(sandbox_path) - return sandbox_path - - def find_datasets(self, query_limit): - """ Find all datasets, and yield them in an endless loop. """ - datasets = Dataset.objects.exclude(dataset_file='', external_path='') - datasets = datasets.order_by('-id').values_list('id', flat=True) - if query_limit >= 0: - datasets = datasets[:query_limit] - while True: - for dataset_id in datasets: - yield dataset_id - if datasets.count() == 0: - raise RuntimeError('No datasets found.') - - def check_dataset(self, dataset, sandbox_path, host): - """ Copy a dataset file, check the MD5, and return a report. - - :return: log_level, message - """ - dest_filename = os.path.join(sandbox_path, - 'ds_{}_{}'.format(dataset.id, dataset.name)) - - if dataset.dataset_file: - source_filename = dataset.dataset_file.path - else: - source_filename = dataset.external_absolute_path() - - if not os.path.exists(source_filename): - return logging.ERROR, 'Dataset file missing: {!r}'.format(source_filename) - - shutil.copyfile(source_filename, dest_filename) - with open(dest_filename, "rb") as f: - new_md5 = compute_md5(f) - - if new_md5 != dataset.MD5_checksum: - message = 'MD5 check failed on {}, dataset id {}: {!r} expected {}, but was {}.'.format( - host, - dataset.id, - source_filename, - dataset.MD5_checksum, - new_md5) - return logging.ERROR, message - return logging.DEBUG, 'MD5 matched for dataset id {}.'.format(dataset.id) - - def purge_trash(self, sandbox_path, limit): - """ Infinite generator function that will purge oldest files. - - :return: a generator that yields None for successful purge, or an error - message. - """ - previous_stats = [] - while True: - filenames = None - try: - filenames = os.listdir(sandbox_path) - if len(filenames) > limit: - filepaths = (os.path.join(sandbox_path, filename) - for filename in filenames) - filestats = sorted((os.stat(path).st_mtime, path) - for path in filepaths) - for _mtime, filename in filestats[:len(filestats)+1-limit]: - os.remove(os.path.join(sandbox_path, filename)) - previous_stats = filestats - yield - except StandardError: - message = '\n'.join(['Purge failed with previous stats:', - repr(previous_stats), - 'current file names:', - repr(sorted(filenames)), - 'error:', - format_exc()]) - yield message - - # noinspection PyArgumentList - def launch_workers(self, worker_count, options): - manage_script = sys.argv[0] - spawn_args = [manage_script, - "copy_and_check_md5s", - "--worker", - "--limit", str(options["limit"])] - mpi_info = MPI.Info.Create() - mpi_info.Set("add-hostfile", "kive/hostfile") - comm = MPI.COMM_SELF.Spawn(sys.executable, - args=spawn_args, - maxprocs=worker_count, - info=mpi_info).Merge() - return comm - - # noinspection PyArgumentList - def run_worker(self, limit): - manager_rank = 0 - comm = MPI.Comm.Get_parent().Merge() - host = MPI.Get_processor_name() - rank = comm.Get_rank() - sandbox_path = self.create_sandbox(rank) - cleaner = self.purge_trash(sandbox_path, limit) - result = (logging.INFO, 'Worker {} started on {}.'.format(rank, host)) - while True: - comm.send(result, dest=manager_rank) - dataset_id = self.polling_receive(comm, source=manager_rank) - try: - dataset = Dataset.objects.get(id=dataset_id) - self.check_dataset(dataset, sandbox_path, host) - purge_message = cleaner.next() - if purge_message is None: - result = (logging.DEBUG, 'Checked dataset id {}.'.format(dataset_id)) - else: - result = (logging.ERROR, purge_message) - except StandardError: - result = (logging.ERROR, format_exc()) diff --git a/kive/librarian/migrations/0110_mark_uploads.py b/kive/librarian/migrations/0110_mark_uploads.py index 566ccfc0f..7d0f99435 100644 --- a/kive/librarian/migrations/0110_mark_uploads.py +++ b/kive/librarian/migrations/0110_mark_uploads.py @@ -9,6 +9,8 @@ def mark_uploads(apps, schema_editor): # noinspection PyPep8Naming Dataset = apps.get_model('librarian', 'Dataset') + if Dataset.objects.count() == 0: + return # noinspection PyPep8Naming ContainerDataset = apps.get_model('container', 'ContainerDataset') # noinspection PyPep8Naming diff --git a/kive/librarian/migrations/0112_remove_dataset_file_source.py b/kive/librarian/migrations/0112_remove_dataset_file_source.py new file mode 100644 index 000000000..19741e8b1 --- /dev/null +++ b/kive/librarian/migrations/0112_remove_dataset_file_source.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.21 on 2019-06-14 22:37 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('librarian', '0111_is_uploaded_false'), + ] + + operations = [ + migrations.RemoveField( + model_name='dataset', + name='file_source', + ), + ] diff --git a/kive/librarian/models.py b/kive/librarian/models.py index 8f2426a66..6424b333e 100644 --- a/kive/librarian/models.py +++ b/kive/librarian/models.py @@ -139,14 +139,6 @@ class Dataset(metadata.models.AccessControl): help_text="Date of Dataset creation.", db_index=True) - # Four cases from which Datasets can originate: - # - # Case 1: uploaded - # Case 2: from the transformation of a RunStep - # Case 3: from the execution of a POC (i.e. from a ROC) - # Case 4: from the execution of a PSIC (i.e. from a RunSIC) - file_source = models.ForeignKey("archive.RunComponent", related_name="outputs", null=True, blank=True) - # Datasets are stored in the "Datasets" folder dataset_file = models.FileField(upload_to=get_upload_path, help_text="Physical path where datasets are stored", @@ -451,10 +443,6 @@ def clean(self): if self.has_structure(): self.structure.clean() - if self.file_source is not None: - # Whatever run created this Dataset must have had access to the parent Dataset. - self.file_source.definite.top_level_run.validate_restrict_access([self]) - if not (self.externalfiledirectory and self.external_path or not self.externalfiledirectory and not self.external_path): raise ValidationError( @@ -496,10 +484,6 @@ def get_access_limits(self, access_limits=None): if access_limits is None: access_limits = [] - # Is this an output from an old run? - if self.file_source is not None: - access_limits.append(self.file_source.parent_run) - # Is this an output from a container run? for container_dataset in self.containers.filter(argument__type='O'): access_limits.append(container_dataset.run) diff --git a/kive/librarian/tests.py b/kive/librarian/tests.py index f56aac42a..5a74d72d6 100644 --- a/kive/librarian/tests.py +++ b/kive/librarian/tests.py @@ -32,8 +32,8 @@ from rest_framework.test import force_authenticate, APIRequestFactory from rest_framework import status -from archive.models import ExecLog, MethodOutput, Run, RunStep, RunComponentState -from constants import datatypes, groups, runcomponentstates +from archive.models import ExecLog, MethodOutput, Run +from constants import datatypes, groups from container.models import ContainerFamily from datachecking.models import MD5Conflict from librarian.ajax import ExternalFileDirectoryViewSet, DatasetViewSet @@ -50,6 +50,8 @@ FROM_FILE_END = 2 +samplecode_path = os.path.abspath(os.path.join(__file__, '../../../samplecode')) + def er_from_record(record): """ @@ -81,7 +83,18 @@ class LibrarianTestCase(TestCase, object): """ def setUp(self): """Set up default database state for librarian unit testing.""" - tools.create_librarian_test_environment(self) + self.myUser = User.objects.create_user('john', + 'lennon@thebeatles.com', + 'johnpassword') + self.ringoUser = User.objects.create_user('ringo', + 'starr@thebeatles.com', + 'ringopassword') + self.singlet_dataset = Dataset.create_dataset( + os.path.join(samplecode_path, "singlet_cdt_large.csv"), + self.myUser, + groups_allowed=[everyone_group()], + name="singlet", + description="lol") def tearDown(self): tools.clean_up_all_files() @@ -186,10 +199,6 @@ def test_filehandle(self): expected_md5 )) - def test_is_raw(self): - self.assertEqual(self.triplet_dataset.is_raw(), False) - self.assertEqual(self.raw_dataset.is_raw(), True) - def test_forgot_header(self): """ Dataset creation with a CDT fails when the header is left off @@ -267,34 +276,6 @@ def test_right_columns(self): Dataset.create_dataset(file_path=file_path, user=self.myUser, cdt=self.cdt_record, description="right columns", name="good data") - def test_invalid_integer_field(self): - compound_datatype = CompoundDatatype(user=self.myUser) - compound_datatype.save() - compound_datatype.members.create(datatype=self.STR, - column_name="name", - column_idx=1) - compound_datatype.members.create(datatype=self.INT, - column_name="count", - column_idx=2) - compound_datatype.clean() - - data_file = dsix.StringIO("""\ -name,count -Bob,tw3nty -""") - data_file.name = 'test_file.csv' - - self.assertRaisesRegexp( - ValueError, - re.escape('The entry at row 1, column 2 of file "{}" did not pass the constraints of Datatype "integer"' - .format(data_file.name)), - lambda: Dataset.create_dataset(file_path=None, - file_handle=data_file, - user=self.myUser, - cdt=compound_datatype, - name="bad data", - description="bad integer field")) - def test_dataset_creation(self): """ Test coherence of a freshly created Dataset. @@ -398,35 +379,6 @@ def test_increase_permissions(self): dataset.refresh_from_db() self.assertTrue(dataset.shared_with_everyone) - def test_source_run_permissions(self): - """ Dataset not allowed to have more permissions than source run. """ - run_step = RunStep.objects.first() - run_step.run.groups_allowed.clear() - - dataset = self.singlet_dataset - dataset.groups_allowed.clear() - dataset.file_source = run_step - dataset.save() - self.assertFalse(dataset.shared_with_everyone) - expected_errors = {'permissions': ['Select a valid choice. Everyone ' - 'is not one of the available ' - 'choices.']} - - user = dataset.user - client = Client() - client.force_login(user) - - response = client.post(reverse('dataset_view', - kwargs=dict(dataset_id=dataset.id)), - dict(name='synglet', - permissions_1='Everyone')) - - self.assertEqual(200, response.status_code) # Form error, not redirect - self.assertEqual(expected_errors, - response.context['dataset_form'].errors) - dataset.refresh_from_db() - self.assertFalse(dataset.shared_with_everyone) - def test_source_container_run_permissions(self): """ Dataset can't have more permissions than source container run. """ user = self.singlet_dataset.user @@ -512,702 +464,25 @@ def test_archive_upload(self): self.assertTrue(dataset1.is_uploaded) -class DatasetStructureTests(LibrarianTestCase): - - def test_num_rows(self): - self.assertEqual(self.triplet_3_rows_dataset.num_rows(), 3) - self.assertEqual(self.triplet_3_rows_dataset.structure.num_rows, 3) - - -@skipIfDBFeature('is_mocked') -class FindCompatibleERTests(TestCase): - fixtures = ['simple_run'] - - def find_run_step(self): - for e in ExecRecord.objects.all(): - if e.has_ever_failed(): - continue - is_running = False - runstep = None - for runcomponent in e.used_by_components.all(): - if type(runcomponent.definite) is RunStep: - runstep = runcomponent.definite - if not runcomponent.top_level_run.is_complete(): - is_running = True - break - if not is_running and runstep: - return runstep - - def test_find_compatible_ER_never_failed(self): - """Should be able to find a compatible ExecRecord which never failed.""" - runstep = self.find_run_step() - execrecord = runstep.execrecord - self.assertIsNotNone(execrecord) - input_datasets_decorated = [(eri.generic_input.definite.dataset_idx, eri.dataset) - for eri in execrecord.execrecordins.all()] - input_datasets_decorated.sort() - input_datasets = [entry[1] for entry in input_datasets_decorated] - runstep.reused = False - runstep.save() - self.assertFalse(execrecord.has_ever_failed()) - self.assertIn(execrecord, runstep.find_compatible_ERs(input_datasets)) - - def test_find_compatible_ER_redacted(self): - """Should not be able to find a redacted ExecRecord.""" - runstep = self.find_run_step() - execrecord = runstep.execrecord - self.assertIsNotNone(execrecord) - execrecord.execrecordins.first().dataset.redact() - input_datasets_decorated = [(eri.generic_input.definite.dataset_idx, eri.dataset) - for eri in execrecord.execrecordins.all()] - input_datasets_decorated.sort() - input_datasets = [entry[1] for entry in input_datasets_decorated] - runstep.reused = False - runstep.save() - self.assertTrue(execrecord.is_redacted()) - self.assertNotIn(execrecord, runstep.find_compatible_ERs(input_datasets)) - - def test_find_compatible_ER_failed(self): - """Should also find a compatible ExecRecord which failed.""" - runstep = self.find_run_step() - execrecord = runstep.execrecord - self.assertIsNotNone(execrecord) - methodoutput = runstep.log.methodoutput - methodoutput.return_code = 1 # make this a failure - methodoutput.save() - # noinspection PyUnresolvedReferences - runstep._runcomponentstate = RunComponentState.objects.get(pk=runcomponentstates.FAILED_PK) - runstep.save() - - input_datasets_decorated = [(eri.generic_input.definite.dataset_idx, eri.dataset) - for eri in execrecord.execrecordins.all()] - input_datasets_decorated.sort() - input_datasets = [entry[1] for entry in input_datasets_decorated] - runstep = execrecord.used_by_components.first().definite - runstep.reused = False - runstep.save() - self.assertTrue(execrecord.has_ever_failed()) - self.assertIn(execrecord, runstep.find_compatible_ERs(input_datasets)) - - def test_find_compatible_ER_skips_nulls(self): - """ - Incomplete run steps don't break search for compatible ExecRecords. - """ - # Find an ExecRecord that has never failed - runstep = self.find_run_step() - execrecord = runstep.execrecord - input_datasets_decorated = [(eri.generic_input.definite.dataset_idx, eri.dataset) - for eri in execrecord.execrecordins.all()] - input_datasets_decorated.sort() - input_datasets = [entry[1] for entry in input_datasets_decorated] - - method = execrecord.general_transf() - pipeline = execrecord.generating_run.pipeline - ps = pipeline.steps.filter(transformation=method).first() - - # Create two RunSteps using this method. First, an incomplete one. - run1 = Run(user=pipeline.user, pipeline=pipeline, name="First incomplete run", - description="Be patient!") - run1.save() - run1.start() - run1.runsteps.create(pipelinestep=ps) - - # Second, one that is looking for an ExecRecord. - run2 = Run(user=pipeline.user, pipeline=pipeline, name="Second run in progress", - description="Impatient!") - run2.save() - run2.start() - rs2 = run2.runsteps.create(pipelinestep=ps) - - self.assertIn(execrecord, rs2.find_compatible_ERs(input_datasets)) - - -@skipIfDBFeature('is_mocked') -class RemovalTests(TestCase): - fixtures = ["removal"] - - def setUp(self): - install_fixture_files("removal") - - self.remover = User.objects.get(username="RemOver") - self.noop_plf = PipelineFamily.objects.get(name="Nucleotide Sequence Noop") - self.noop_pl = self.noop_plf.members.get(revision_name="v1") - self.first_run = self.noop_pl.pipeline_instances.order_by("start_time").first() - self.second_run = self.noop_pl.pipeline_instances.order_by("start_time").last() - self.input_DS = Dataset.objects.get(name="Removal test data") - self.nuc_seq_noop_mf = MethodFamily.objects.get(name="Noop (nucleotide sequence)") - self.nuc_seq_noop = self.nuc_seq_noop_mf.members.get(revision_name="v1") - self.p_nested_plf = PipelineFamily.objects.get(name="Nested pipeline") - self.p_nested = self.p_nested_plf.members.get(revision_name="v1") - self.noop_cr = CodeResource.objects.get(name="Noop") - self.noop_crr = self.noop_cr.revisions.get(revision_name="1") - self.pass_through_cr = CodeResource.objects.get(name="Pass Through") - self.pass_through_crr = self.pass_through_cr.revisions.get(revision_name="1") - self.raw_pass_through_mf = MethodFamily.objects.get(name="Pass-through (raw)") - self.raw_pass_through = self.raw_pass_through_mf.members.get(revision_name="v1") - self.nuc_seq = Datatype.objects.get(name="Nucleotide sequence") - self.one_col_nuc_seq = self.nuc_seq.CDTMs.get(column_name="sequence", column_idx=1).compounddatatype - - self.two_step_noop_plf = PipelineFamily.objects.get(name="Nucleotide Sequence two-step Noop") - self.two_step_noop_pl = self.two_step_noop_plf.members.get(revision_name="v1") - self.two_step_input_dataset = Dataset.objects.get(name="Removal test data for a two-step Pipeline") - - # Datasets and ExecRecords produced by the first run. - self.produced_data = set() - self.execrecords = set() - for runstep in self.first_run.runsteps.all(): - self.produced_data.update(runstep.outputs.all()) - self.execrecords.add(runstep.execrecord) - for rsic in runstep.RSICs.all(): - self.produced_data.update(rsic.outputs.all()) - self.execrecords.add(rsic.execrecord) - for roc in self.first_run.runoutputcables.all(): - self.produced_data.update(roc.outputs.all()) - self.execrecords.add(roc.execrecord) - - self.step_log = self.first_run.runsteps.first().log - - self.two_step_run = self.two_step_noop_pl.pipeline_instances.first() - self.two_step_intermediate_data = self.two_step_run.runsteps.get( - pipelinestep__step_num=1).outputs.first() - self.two_step_output_data = self.two_step_run.runsteps.get( - pipelinestep__step_num=2).outputs.first() - self.two_step_execrecords = set() - for runstep in self.two_step_run.runsteps.all(): - self.two_step_execrecords.add(runstep.execrecord) - for rsic in runstep.RSICs.all(): - self.two_step_execrecords.add(rsic.execrecord) - for roc in self.two_step_run.runoutputcables.all(): - self.two_step_execrecords.add(roc.execrecord) - - def tearDown(self): - tools.clean_up_all_files() - remove_fixture_files() - - def removal_plan_tester(self, obj_to_remove, datasets=None, ers=None, runs=None, pipelines=None, pfs=None, - methods=None, mfs=None, cdts=None, dts=None, crrs=None, crs=None, - external_files=None): - removal_plan = obj_to_remove.build_removal_plan() - self.assertSetEqual(removal_plan["Datasets"], set(datasets) if datasets is not None else set()) - self.assertSetEqual(removal_plan["ExecRecords"], set(ers) if ers is not None else set()) - self.assertSetEqual(removal_plan["Runs"], set(runs) if runs is not None else set()) - self.assertSetEqual(removal_plan["Pipelines"], set(pipelines) if pipelines is not None else set()) - self.assertSetEqual(removal_plan["PipelineFamilies"], set(pfs) if pfs is not None else set()) - self.assertSetEqual(removal_plan["Methods"], set(methods) if methods is not None else set()) - self.assertSetEqual(removal_plan["MethodFamilies"], set(mfs) if mfs is not None else set()) - self.assertSetEqual(removal_plan["CompoundDatatypes"], set(cdts) if cdts is not None else set()) - self.assertSetEqual(removal_plan["Datatypes"], set(dts) if dts is not None else set()) - self.assertSetEqual(removal_plan["CodeResourceRevisions"], set(crrs) if crrs is not None else set()) - self.assertSetEqual(removal_plan["CodeResources"], set(crs) if crs is not None else set()) - self.assertSetEqual(removal_plan["ExternalFiles"], set(external_files) if external_files is not None else set()) - - def test_run_build_removal_plan(self): - """Removing a Run should remove all intermediate/output data and ExecRecords, and all Runs that reused it.""" - self.removal_plan_tester(self.first_run, datasets=self.produced_data, ers=self.execrecords, - runs={self.first_run, self.second_run}) - - def test_reused_run_build_removal_plan(self): - """Removing a reused Run should leave reused data/ExecRecords alone.""" - self.removal_plan_tester(self.second_run, runs={self.second_run}) - - def test_input_data_build_removal_plan(self): - """Removing input data to a Run should remove any Run started from it.""" - self.removal_plan_tester( - self.input_DS, - datasets=self.produced_data.union({self.input_DS}), - ers=self.execrecords, - runs={self.first_run, self.second_run} - ) - - def test_external_input_build_removal_plan(self): - """Removing an input dataset that is externally-backed.""" - working_dir = tempfile.mkdtemp() - efd = ExternalFileDirectory( - name="TestBuildRemovalPlanEFD", - path=working_dir - ) - efd.save() - - ext_path = "ext.txt" - self.input_DS.dataset_file.open() - with self.input_DS.dataset_file: - with open(os.path.join(working_dir, ext_path), "wb") as f: - f.write(self.input_DS.dataset_file.read()) - - # Mark the input dataset as externally-backed. - self.input_DS.externalfiledirectory = efd - self.input_DS.external_path = ext_path - self.input_DS.save() - - all_data = self.produced_data - all_data.add(self.input_DS) - - self.removal_plan_tester( - self.input_DS, - datasets=self.produced_data.union({self.input_DS}), - ers=self.execrecords, - runs={self.first_run, self.second_run}, - external_files={self.input_DS} - ) - - def test_produced_data_build_removal_plan(self): - """Removing data produced by the Run should have the same effect as removing the Run itself.""" - produced_dataset = list(self.produced_data)[0] - - self.removal_plan_tester(produced_dataset, datasets=self.produced_data, ers=self.execrecords, - runs={self.first_run, self.second_run}) - - def test_step_ER_build_removal_plan(self): - """Removing the ExecRecord of the first RunStep should be like removing the whole Run.""" - first_step_er = self.first_run.runsteps.get(pipelinestep__step_num=1).execrecord - - self.removal_plan_tester(first_step_er, datasets=self.produced_data, ers=self.execrecords, - runs={self.first_run, self.second_run}) - - def test_rsic_ER_build_removal_plan(self): - """Removing the ExecRecord of a RunSIC should be like removing the whole Run.""" - first_rsic_er = self.first_run.runsteps.get(pipelinestep__step_num=1).RSICs.first().execrecord - - self.removal_plan_tester(first_rsic_er, datasets=self.produced_data, ers=self.execrecords, - runs={self.first_run, self.second_run}) - - def test_roc_ER_build_removal_plan(self): - """Removing the ExecRecord of a RunOutputCable should be like removing the whole Run.""" - first_roc_er = self.first_run.runoutputcables.first().execrecord - - self.removal_plan_tester(first_roc_er, datasets=self.produced_data, ers=self.execrecords, - runs={self.first_run, self.second_run}) - - def test_pipeline_build_removal_plan(self): - """Removing a Pipeline.""" - self.removal_plan_tester(self.noop_pl, datasets=self.produced_data, ers=self.execrecords, - runs={self.first_run, self.second_run}, pipelines={self.noop_pl, self.p_nested}) - - def test_nested_pipeline_build_removal_plan(self): - """Removing a nested Pipeline.""" - self.removal_plan_tester(self.p_nested, pipelines={self.p_nested}) - - def test_pipelinefamily_build_removal_plan(self): - """Removing a PipelineFamily removes everything that goes along with it.""" - self.removal_plan_tester(self.noop_plf, datasets=self.produced_data, ers=self.execrecords, - runs={self.first_run, self.second_run}, pipelines={self.noop_pl, self.p_nested}, - pfs={self.noop_plf}) - - def test_method_build_removal_plan(self): - """Removing a Method removes all Pipelines containing it and all of the associated stuff.""" - self.removal_plan_tester( - self.nuc_seq_noop, - datasets=self.produced_data.union({self.two_step_intermediate_data, self.two_step_output_data}), - ers=self.execrecords.union(self.two_step_execrecords), - runs={self.first_run, self.second_run, self.two_step_run}, - pipelines={self.noop_pl, self.p_nested, self.two_step_noop_pl}, - methods={self.nuc_seq_noop} - ) - - def test_methodfamily_build_removal_plan(self): - """Removing a MethodFamily.""" - self.removal_plan_tester( - self.nuc_seq_noop_mf, - datasets=self.produced_data.union( - {self.two_step_intermediate_data, self.two_step_output_data} - ), - ers=self.execrecords.union(self.two_step_execrecords), - runs={self.first_run, self.second_run, self.two_step_run}, - pipelines={self.noop_pl, self.p_nested, self.two_step_noop_pl}, - methods={self.nuc_seq_noop}, - mfs={self.nuc_seq_noop_mf} - ) - - def test_crr_build_removal_plan(self): - """Removing a CodeResourceRevision.""" - self.removal_plan_tester( - self.noop_crr, - datasets=self.produced_data.union({self.two_step_intermediate_data, self.two_step_output_data}), - ers=self.execrecords.union(self.two_step_execrecords), - runs={self.first_run, self.second_run, self.two_step_run}, - pipelines={self.noop_pl, self.p_nested, self.two_step_noop_pl}, - methods={self.nuc_seq_noop, self.raw_pass_through}, - crrs={self.noop_crr} - ) - - def test_method_nodep_build_removal_plan(self): - """Removing a Method that has CodeResourceDependencies leaves it alone.""" - self.removal_plan_tester(self.raw_pass_through, methods={self.raw_pass_through}) - - def test_cr_build_removal_plan(self): - """Removing a CodeResource removes its revisions.""" - self.removal_plan_tester( - self.noop_cr, - datasets=self.produced_data.union({self.two_step_intermediate_data, self.two_step_output_data}), - ers=self.execrecords.union(self.two_step_execrecords), - runs={self.first_run, self.second_run, self.two_step_run}, - pipelines={self.noop_pl, self.p_nested, self.two_step_noop_pl}, - methods={self.nuc_seq_noop, self.raw_pass_through}, - crrs={self.noop_crr}, - crs={self.noop_cr} - ) - - def test_cdt_build_removal_plan(self): - """Removing a CompoundDatatype.""" - all_data = self.produced_data.union( - { - self.input_DS, - self.two_step_input_dataset, - self.two_step_intermediate_data, - self.two_step_output_data - } - ) - self.removal_plan_tester( - self.one_col_nuc_seq, - datasets=all_data, - ers=self.execrecords.union(self.two_step_execrecords), - runs={self.first_run, self.second_run, self.two_step_run}, - pipelines={self.noop_pl, self.p_nested, self.two_step_noop_pl}, - methods={self.nuc_seq_noop}, - cdts={self.one_col_nuc_seq} - ) - - def test_dt_build_removal_plan(self): - """Removing a Datatype.""" - all_data = self.produced_data.union( - { - self.input_DS, - self.two_step_input_dataset, - self.two_step_intermediate_data, - self.two_step_output_data - } - ) - self.removal_plan_tester( - self.nuc_seq, - datasets=all_data, - ers=self.execrecords.union(self.two_step_execrecords), - runs={self.first_run, self.second_run, self.two_step_run}, - pipelines={self.noop_pl, self.p_nested, self.two_step_noop_pl}, - methods={self.nuc_seq_noop}, - cdts={self.one_col_nuc_seq}, - dts={self.nuc_seq} - ) - - def remove_tester(self, obj_to_remove): - removal_plan = obj_to_remove.build_removal_plan() - - dataset_pks = [x.pk for x in removal_plan["Datasets"]] - er_pks = [x.pk for x in removal_plan["ExecRecords"]] - run_pks = [x.pk for x in removal_plan["Runs"]] - pipeline_pks = [x.pk for x in removal_plan["Pipelines"]] - pf_pks = [x.pk for x in removal_plan["PipelineFamilies"]] - method_pks = [x.pk for x in removal_plan["Methods"]] - mf_pks = [x.pk for x in removal_plan["MethodFamilies"]] - cdt_pks = [x.pk for x in removal_plan["CompoundDatatypes"]] - dt_pks = [x.pk for x in removal_plan["Datatypes"]] - crr_pks = [x.pk for x in removal_plan["CodeResourceRevisions"]] - cr_pks = [x.pk for x in removal_plan["CodeResources"]] - - obj_to_remove.remove() - self.assertFalse(Dataset.objects.filter(pk__in=dataset_pks).exists()) - self.assertFalse(ExecRecord.objects.filter(pk__in=er_pks).exists()) - self.assertFalse(Run.objects.filter(pk__in=run_pks).exists()) - self.assertFalse(Pipeline.objects.filter(pk__in=pipeline_pks).exists()) - self.assertFalse(PipelineFamily.objects.filter(pk__in=pf_pks).exists()) - self.assertFalse(Method.objects.filter(pk__in=method_pks).exists()) - self.assertFalse(MethodFamily.objects.filter(pk__in=mf_pks).exists()) - self.assertFalse(CompoundDatatype.objects.filter(pk__in=cdt_pks).exists()) - self.assertFalse(Datatype.objects.filter(pk__in=dt_pks).exists()) - self.assertFalse(CodeResourceRevision.objects.filter(pk__in=crr_pks).exists()) - self.assertFalse(CodeResource.objects.filter(pk__in=cr_pks).exists()) - - def test_pipeline_remove(self): - """ - Removing a Pipeline should remove all Runs created from it. - """ - self.remove_tester(self.noop_pl) - - def test_nested_pipeline_remove(self): - """Removing a nested Pipeline.""" - self.remove_tester(self.p_nested) - - def test_pipelinefamily_remove(self): - """Removing a PipelineFamily should remove all Pipelines in it.""" - self.remove_tester(self.noop_plf) - - def test_method_remove(self): - """Removing a Method should remove the Pipelines containing it.""" - - self.remove_tester(self.nuc_seq_noop) - - def test_methodfamily_remove(self): - """Removing a MethodFamily should remove the Methods in it.""" - self.remove_tester(self.nuc_seq_noop_mf) - - def test_crr_remove(self): - """Removing a CodeResourceRevision should remove the Methods using it, and its dependencies.""" - self.remove_tester(self.noop_crr) - - def test_method_nodep_remove(self): - """Removing a Method that has dependencies leaves the dependencies alone.""" - self.remove_tester(self.raw_pass_through) - - def test_cr_remove(self): - """Removing a CodeResource should remove the CodeResourceRevisions using it.""" - self.remove_tester(self.noop_cr) - - def test_cdt_remove(self): - """Removing a CDT should remove the Methods/Pipelines/Datasets using it.""" - self.remove_tester(self.one_col_nuc_seq) - - def test_datatype_remove(self): - """Removing a Datatype should remove the CDTs that use it.""" - self.remove_tester(self.nuc_seq) - - def test_dataset_remove(self): - """Removing a Dataset should remove anything that touches it.""" - self.remove_tester(self.input_DS) - - def test_run_remove(self): - """Removing a Run.""" - self.remove_tester(self.first_run) - - def test_reused_run_remove(self): - """Removing a reused Run.""" - self.remove_tester(self.second_run) - - def test_produced_data_remove(self): - """Removing data produced by the Run should have the same effect as removing the Run itself.""" - produced_dataset = list(self.produced_data)[0] - self.remove_tester(produced_dataset) - - def test_step_ER_remove(self): - """Removing the ExecRecord of the first RunStep should be like removing the whole Run.""" - first_step_er = self.first_run.runsteps.get(pipelinestep__step_num=1).execrecord - self.remove_tester(first_step_er) - - def test_rsic_ER_remove(self): - """Removing the ExecRecord of a RunSIC should be like removing the whole Run.""" - first_rsic_er = self.first_run.runsteps.get(pipelinestep__step_num=1).RSICs.first().execrecord - self.remove_tester(first_rsic_er) - - def test_roc_ER_remove(self): - """Removing the ExecRecord of a RunOutputCable should be like removing the whole Run.""" - first_roc_er = self.first_run.runoutputcables.first().execrecord - self.remove_tester(first_roc_er) - - def dataset_redaction_plan_tester(self, dataset_to_redact, datasets=None, output_logs=None, error_logs=None, - return_codes=None, external_files=None): - redaction_plan = dataset_to_redact.build_redaction_plan() - - # The following ExecRecords should also be in the redaction plan. - redaction_plan_execrecords = set() - dataset_set = datasets or set() - for dataset in dataset_set: - for eri in dataset.execrecordins.all(): - redaction_plan_execrecords.add(eri.execrecord) - - self.assertSetEqual(redaction_plan["Datasets"], set(datasets) if datasets is not None else set()) - self.assertSetEqual(redaction_plan["OutputLogs"], set(output_logs) if output_logs is not None else set()) - self.assertSetEqual(redaction_plan["ErrorLogs"], set(error_logs) if error_logs is not None else set()) - self.assertSetEqual(redaction_plan["ReturnCodes"], set(return_codes) if return_codes is not None else set()) - self.assertSetEqual(redaction_plan["ExecRecords"], redaction_plan_execrecords) - self.assertSetEqual(redaction_plan["ExternalFiles"], - set(external_files) if external_files is not None else set()) - - def dataset_redaction_tester(self, dataset_to_redact): - redaction_plan = dataset_to_redact.build_redaction_plan() - dataset_to_redact.redact() - self.redaction_tester_helper(redaction_plan) - - def redaction_tester_helper(self, redaction_plan): - # Check that all of the objects in the plan, and the RunComponents/ExecRecords that - # reference them, got redacted. - for dataset in redaction_plan["Datasets"]: - reloaded_dataset = Dataset.objects.get(pk=dataset.pk) - self.assertTrue(reloaded_dataset.is_redacted()) - - execlogs_affected = redaction_plan["OutputLogs"].union( - redaction_plan["ErrorLogs"]).union(redaction_plan["ReturnCodes"]) - for log in execlogs_affected: - # noinspection PyUnresolvedReferences - reloaded_log = ExecLog.objects.get(pk=log.pk) - if log in redaction_plan["OutputLogs"]: - self.assertTrue(reloaded_log.methodoutput.is_output_redacted()) - if log in redaction_plan["ErrorLogs"]: - self.assertTrue(reloaded_log.methodoutput.is_error_redacted()) - if log in redaction_plan["ReturnCodes"]: - self.assertTrue(reloaded_log.methodoutput.is_code_redacted()) - - self.assertTrue(reloaded_log.is_redacted()) - self.assertTrue(reloaded_log.record.is_redacted()) - if reloaded_log.generated_execrecord(): - self.assertTrue(reloaded_log.execrecord.is_redacted()) - - for er in redaction_plan["ExecRecords"]: - self.assertTrue(er.is_redacted()) - for rc in er.used_by_components.all(): - self.assertTrue(rc.is_redacted()) - - def log_redaction_plan_tester(self, log_to_redact, output_log=True, error_log=True, return_code=True): - output_already_redacted = log_to_redact.methodoutput.is_output_redacted() - error_already_redacted = log_to_redact.methodoutput.is_error_redacted() - code_already_redacted = log_to_redact.methodoutput.is_code_redacted() - - redaction_plan = log_to_redact.build_redaction_plan(output_log=output_log, error_log=error_log, - return_code=return_code) - - self.assertSetEqual(redaction_plan["Datasets"], set()) - self.assertSetEqual(redaction_plan["ExecRecords"], set()) - self.assertSetEqual(redaction_plan["OutputLogs"], - {log_to_redact} if output_log and not output_already_redacted else set()) - self.assertSetEqual(redaction_plan["ErrorLogs"], - {log_to_redact} if error_log and not error_already_redacted else set()) - self.assertSetEqual(redaction_plan["ReturnCodes"], - {log_to_redact} if return_code and not code_already_redacted else set()) - - def log_redaction_tester(self, log_to_redact, output_log=True, error_log=True, return_code=True): - redaction_plan = log_to_redact.build_redaction_plan(output_log, error_log, return_code) - - if output_log: - log_to_redact.methodoutput.redact_output_log() - if error_log: - log_to_redact.methodoutput.redact_error_log() - if return_code: - log_to_redact.methodoutput.redact_return_code() - - self.redaction_tester_helper(redaction_plan) - - def test_input_dataset_build_redaction_plan(self): - """Test redaction of the input dataset to a Run.""" - logs_to_redact = {self.step_log} - - self.dataset_redaction_plan_tester( - self.input_DS, - datasets=self.produced_data.union({self.input_DS}), - output_logs=logs_to_redact, - error_logs=logs_to_redact, - return_codes=logs_to_redact - ) - - def test_external_input_build_redaction_plan(self): - """Redacting an input dataset that is externally-backed.""" - working_dir = tempfile.mkdtemp() - efd = ExternalFileDirectory( - name="TestBuildRemovalPlanEFD", - path=working_dir - ) - efd.save() - - ext_path = "ext.txt" - self.input_DS.dataset_file.open() - with self.input_DS.dataset_file: - with open(os.path.join(working_dir, ext_path), "wb") as f: - f.write(self.input_DS.dataset_file.read()) - - # Mark the input dataset as externally-backed. - self.input_DS.externalfiledirectory = efd - self.input_DS.external_path = ext_path - self.input_DS.save() - - logs_to_redact = {self.step_log} - - self.dataset_redaction_plan_tester( - self.input_DS, - datasets=self.produced_data.union({self.input_DS}), - output_logs=logs_to_redact, - error_logs=logs_to_redact, - return_codes=logs_to_redact, - external_files={self.input_DS} - ) - - def test_input_dataset_redact(self): - self.dataset_redaction_tester(self.input_DS) - - def test_dataset_redact_idempotent(self): - """Redacting an already-redacted Dataset should give an empty redaction plan.""" - self.input_DS.redact() - # All of the parameters to this function are None, indicating nothing gets redacted. - self.dataset_redaction_plan_tester(self.input_DS) - - def test_produced_dataset_build_redaction_plan(self): - """Redacting produced data.""" - # The run we're dealing with has a single step, and that's the only produced data. - produced_dataset = list(self.produced_data)[0] - - self.dataset_redaction_plan_tester( - produced_dataset, - datasets=self.produced_data - ) - - def test_produced_dataset_redact(self): - produced_dataset = list(self.produced_data)[0] - self.dataset_redaction_tester(produced_dataset) - - def test_intermediate_dataset_build_redaction_plan(self): - """Redacting a Dataset from the middle of a Run only redacts the stuff following it.""" - logs_to_redact = {self.two_step_run.runsteps.get(pipelinestep__step_num=2).log} - - self.dataset_redaction_plan_tester( - self.two_step_intermediate_data, - datasets={self.two_step_intermediate_data, self.two_step_output_data}, - output_logs=logs_to_redact, - error_logs=logs_to_redact, - return_codes=logs_to_redact - ) - - def test_intermediate_dataset_redact(self): - self.dataset_redaction_tester(self.two_step_intermediate_data) - - def test_step_log_build_redaction_plan_remove_all(self): - # There's only one step in self.first_run. - self.log_redaction_plan_tester( - self.step_log, True, True, True - ) - - def test_step_log_redact_all(self): - self.log_redaction_tester( - self.step_log, True, True, True - ) - - def test_step_log_build_redaction_plan_redact_output_log(self): - self.log_redaction_plan_tester( - self.step_log, output_log=True, error_log=False, return_code=False - ) - - def test_step_log_redact_output_log(self): - self.log_redaction_tester( - self.step_log, output_log=True, error_log=False, return_code=False - ) - - def test_step_log_build_redaction_plan_redact_error_log(self): - self.log_redaction_plan_tester( - self.step_log, output_log=False, error_log=True, return_code=False - ) - - def test_step_log_redact_error_log(self): - self.log_redaction_tester( - self.step_log, output_log=False, error_log=True, return_code=False - ) - - def test_step_log_build_redaction_plan_redact_return_code(self): - self.log_redaction_plan_tester( - self.step_log, output_log=False, error_log=False, return_code=True - ) - - def test_step_log_redact_return_code(self): - self.log_redaction_tester( - self.step_log, output_log=False, error_log=False, return_code=True - ) - - def test_step_log_build_redaction_plan_redact_partially_redacted(self): - """Redacting something that's been partially redacted should take that into account.""" - self.step_log.methodoutput.redact_output_log() - self.log_redaction_plan_tester( - self.step_log, output_log=True, error_log=True, return_code=True - ) - - @skipIfDBFeature('is_mocked') class DatasetWithFileTests(TestCase): def setUp(self): - tools.create_librarian_test_environment(self) + self.myUser = User.objects.create_user('john', + 'lennon@thebeatles.com', + 'johnpassword') + self.singlet_dataset = Dataset.create_dataset( + os.path.join(samplecode_path, "singlet_cdt_large.csv"), + self.myUser, + groups_allowed=[everyone_group()], + name="singlet", + description="lol") + self.raw_dataset = Dataset.create_dataset( + os.path.join(samplecode_path, "step_0_raw.fasta"), + user=self.myUser, + groups_allowed=[everyone_group()], + name="raw_DS", + description="lol") def tearDown(self): tools.clean_up_all_files() @@ -1283,7 +558,6 @@ def setUp(self): bananas = Dataset(pk=44, name='bananas', date_created=bananas_date, - file_source=RunStep(), user=self.kive_kive_user) Dataset.objects.add(apples, cherries, @@ -1613,24 +887,15 @@ def setUp(self): self.factory = APIRequestFactory() self.list_path = reverse("dataset-list") - # This defines a user named "john" which is now accessible as self.myUser. - tools.create_metadata_test_environment(self) + self.myUser = User.objects.create_user('john', + 'lennon@thebeatles.com', + 'johnpassword') self.kive_user = kive_user() self.duck_context = DuckContext() num_cols = 12 self.raw_file_contents = ','.join(map(str, range(num_cols))).encode() - # A CompoundDatatype that belongs to the Kive user. - self.kive_CDT = CompoundDatatype(user=self.kive_user) - self.kive_CDT.save() - self.kive_CDT.members.create( - datatype=self.string_dt, - column_name="col1", - column_idx=1 - ) - self.kive_CDT.full_clean() - self.kive_file_contents = """col1 foo bar @@ -1714,41 +979,6 @@ def test_validate_with_groups_allowed(self): ) self.assertTrue(ds.is_valid()) - def test_validate_with_CDT(self): - """ - Test validating a Dataset with a CDT. - """ - with tempfile.TemporaryFile(self.csv_file_temp_open_mode) as f: - f.write(self.kive_file_contents) - f.seek(0) - - self.data_to_serialize["dataset_file"] = File(f, name="bla") - self.data_to_serialize["compounddatatype"] = self.kive_CDT.pk - - ds = DatasetSerializer( - data=self.data_to_serialize, - context=self.duck_context - ) - self.assertTrue(ds.is_valid()) - - def test_validate_ineligible_CDT(self): - """ - Test validating a Dataset with a CDT that the user doesn't have access to. - """ - with tempfile.TemporaryFile(self.csv_file_temp_open_mode) as f: - f.write(self.kive_file_contents) - f.seek(0) - - self.data_to_serialize["dataset_file"] = File(f, name="bla") - self.data_to_serialize["compounddatatype"] = self.kive_CDT.pk - - ds = DatasetSerializer( - data=self.data_to_serialize, - context=DuckContext(self.myUser) - ) - self.assertFalse(ds.is_valid()) - self.assertEquals(len(ds.errors["compounddatatype"]), 1) - def test_validate_externally_backed(self): """ Test validating a new Dataset with external backing. @@ -1862,27 +1092,6 @@ def test_create_do_not_retain(self): self.assertEquals(dataset.user, self.kive_user) self.assertFalse(bool(dataset.dataset_file)) - def test_create_with_CDT(self): - """ - Test creating a Dataset with a CDT. - """ - with tempfile.TemporaryFile(mode="w+t") as f: - f.write(self.kive_file_contents) - f.seek(0) - - self.data_to_serialize["dataset_file"] = File(f, name="bla") - self.data_to_serialize["compounddatatype"] = self.kive_CDT.pk - - ds = DatasetSerializer( - data=self.data_to_serialize, - context=self.duck_context - ) - ds.is_valid() - dataset = ds.save() - - # Probe to make sure the CDT got set correctly. - self.assertEquals(dataset.compounddatatype, self.kive_CDT) - def test_create_with_users_allowed(self): """ Test validating a new Dataset with users allowed. @@ -2091,7 +1300,9 @@ def test_list_files(self, mock_walk): class ExternalFileTests(TestCase): def setUp(self): - tools.create_metadata_test_environment(self) + self.myUser = User.objects.create_user('john', + 'lennon@thebeatles.com', + 'johnpassword') self.working_dir = tempfile.mkdtemp() self.efd = ExternalFileDirectory( diff --git a/kive/librarian/tests_mock.py b/kive/librarian/tests_mock.py index 5adc27923..4efb8e202 100644 --- a/kive/librarian/tests_mock.py +++ b/kive/librarian/tests_mock.py @@ -5,12 +5,10 @@ from zipfile import ZipFile from django.contrib.auth.models import Group, User -from django.core.exceptions import ValidationError from django.core.files.base import ContentFile from django.core.files.uploadedfile import SimpleUploadedFile from django.core.urlresolvers import reverse from django.test import TestCase -from django_mock_queries.query import MockSet from django_mock_queries.mocks import mocked_relations import mock from mock import PropertyMock, Mock, patch @@ -20,14 +18,9 @@ from datachecking.models import BadData, CellError, ContentCheckLog from kive.tests import dummy_file, strip_removal_plan from kive.tests import ViewMockTestCase -from librarian.models import Dataset, ExecRecord, ExecRecordOut, ExecRecordIn, DatasetStructure +from librarian.models import Dataset, ExecRecord, ExecRecordOut from metadata.models import Datatype, CompoundDatatypeMember, CompoundDatatype, kive_user, KiveUser -from archive.models import RunStep, ExecLog, Run, RunOutputCable, RunSIC, RunInput, RunComponent -from method.models import Method -from pipeline.models import PipelineOutputCable, PipelineStepInputCable, PipelineStep, Pipeline, PipelineCable, \ - CustomCableWire -from transformation.models import TransformationOutput, XputStructure, TransformationXput, TransformationInput, \ - Transformation +from archive.models import RunStep, ExecLog, Run, RunInput @mocked_relations(Dataset, @@ -351,15 +344,12 @@ def test_dataset_view(self): def test_dataset_view_output(self): """ Link back to the run that generated the output dataset. """ user = User() - run = Run(id=2000, user=user) - self.dataset.file_source = RunStep(run=run) response = self.client.get(reverse('dataset_view', kwargs=dict(dataset_id='99'))) self.assertEqual(200, response.status_code) self.assertEqual(self.file_content, response.context['sample_content']) self.assertEqual('/datasets', response.context['return']) - self.assertIs(run, response.context['generating_run']) def test_dataset_view_run(self): response = self.client.get(reverse('dataset_view', @@ -447,7 +437,6 @@ def test_datasets_add_bulk(self, register_file, compute_md5): @mocked_relations(RunInput) def test_dataset_lookup_not_found(self): - self.dataset.file_source = RunStep() md5_checksum = '123456789012345678901234567890ab' response = self.client.get(reverse( 'dataset_lookup', @@ -460,7 +449,6 @@ def test_dataset_lookup_not_found(self): @mocked_relations(RunInput) def test_dataset_lookup(self): - self.dataset.file_source = RunStep() md5_checksum = '123456789012345678901234567890ab' self.dataset.MD5_checksum = md5_checksum @@ -478,636 +466,3 @@ def test_lookup(self): response = self.client.get(reverse('lookup')) self.assertEqual(200, response.status_code) - - -# noinspection PyUnresolvedReferences -@mocked_relations(ExecRecord) -class ExecRecordQuarantineDecontaminateMockTests(TestCase): - """ - Tests of the quarantine/decontamination functionality of ExecRecord. - """ - def test_quarantine_runcomponents(self): - """ - Quarantines all Successful RunComponents using this ExecRecord. - """ - generating_el = ExecLog() - er = ExecRecord(generator=generating_el) - rs1 = RunStep(execrecord=er, _runcomponentstate_id=runcomponentstates.SUCCESSFUL_PK) - rs2 = RunStep(execrecord=er, _runcomponentstate_id=runcomponentstates.RUNNING_PK) - rs3 = RunStep(execrecord=er, _runcomponentstate_id=runcomponentstates.SUCCESSFUL_PK) - - er.used_by_components.add(rs1, rs2, rs3) - - rs1.quarantine = Mock() - rs2.quarantine = Mock() - rs3.quarantine = Mock() - - er.quarantine_runcomponents() - rs1.quarantine.assert_called_once_with(save=True, recurse_upward=True) - rs2.quarantine.assert_not_called() - rs3.quarantine.assert_called_once_with(save=True, recurse_upward=True) - - def test_decontaminate_runcomponents(self): - """ - Decontaminates all Quarantined RunComponents using this ExecRecord. - """ - generating_el = ExecLog() - er = ExecRecord(generator=generating_el) - rs1 = RunStep(execrecord=er, _runcomponentstate_id=runcomponentstates.QUARANTINED_PK) - rs2 = RunStep(execrecord=er, _runcomponentstate_id=runcomponentstates.RUNNING_PK) - rs3 = RunStep(execrecord=er, _runcomponentstate_id=runcomponentstates.QUARANTINED_PK) - - er.used_by_components.add(rs1, rs2, rs3) - - rs1.decontaminate = Mock() - rs2.decontaminate = Mock() - rs3.decontaminate = Mock() - - er.decontaminate_runcomponents() - rs1.decontaminate.assert_called_once_with(save=True, recurse_upward=True) - rs2.decontaminate.assert_not_called() - rs3.decontaminate.assert_called_once_with(save=True, recurse_upward=True) - - def test_attempt_decontamination(self): - """ - ExecRecord correctly decontaminates all RunComponents using it. - """ - generating_el = ExecLog() - er = ExecRecord(generator=generating_el) - - ds1 = Dataset() - ds2 = Dataset() - ds3 = Dataset() - ero1 = ExecRecordOut(execrecord=er, dataset=ds1) - ero2 = ExecRecordOut(execrecord=er, dataset=ds2) - ero3 = ExecRecordOut(execrecord=er, dataset=ds3) - ero1.is_OK = Mock(return_value=True) - ero2.is_OK = Mock(return_value=True) - ero3.is_OK = Mock(return_value=True) - er.execrecordouts.add(ero1, ero2, ero3) - - rs1 = RunStep(execrecord=er, _runcomponentstate_id=runcomponentstates.QUARANTINED_PK, - end_time=datetime(2000, 2, 14)) - rs1.log = ExecLog(record=rs1) - rs2 = RunStep(execrecord=er, _runcomponentstate_id=runcomponentstates.QUARANTINED_PK, - end_time=datetime(2000, 2, 15)) - rs2.log = ExecLog(record=rs2) - rs3 = RunStep(execrecord=er, _runcomponentstate_id=runcomponentstates.SUCCESSFUL_PK, - end_time=datetime(2000, 2, 16)) - rs3.log = ExecLog(record=rs3) - rs3.log.is_successful = Mock(return_value=True) - er.used_by_components.add(rs1, rs2, rs3) - - er.decontaminate_runcomponents = Mock() - - er.attempt_decontamination(ds1) - ero1.is_OK.assert_not_called() - ero2.is_OK.assert_called_once_with() - ero3.is_OK.assert_called_once_with() - rs3.log.is_successful.assert_called_once_with() - er.decontaminate_runcomponents.assert_called_once_with() - - def test_attempt_decontamination_still_has_bad_outputs(self): - """ - Attempt bails if another output is still bad. - """ - generating_el = ExecLog() - er = ExecRecord(generator=generating_el) - - ds1 = Dataset() - ds2 = Dataset() - ds3 = Dataset() - ero1 = ExecRecordOut(execrecord=er, dataset=ds1) - ero2 = ExecRecordOut(execrecord=er, dataset=ds2) - ero3 = ExecRecordOut(execrecord=er, dataset=ds3) - ero1.is_OK = Mock(return_value=True) - ero2.is_OK = Mock(return_value=False) - ero3.is_OK = Mock(return_value=True) - er.execrecordouts.add(ero1, ero2, ero3) - - rs1 = RunStep(execrecord=er, _runcomponentstate_id=runcomponentstates.QUARANTINED_PK, - end_time=datetime(2000, 2, 14)) - rs1.log = ExecLog(record=rs1) - rs2 = RunStep(execrecord=er, _runcomponentstate_id=runcomponentstates.QUARANTINED_PK, - end_time=datetime(2000, 2, 15)) - rs2.log = ExecLog(record=rs2) - rs3 = RunStep(execrecord=er, _runcomponentstate_id=runcomponentstates.SUCCESSFUL_PK, - end_time=datetime(2000, 2, 16)) - rs3.log = ExecLog(record=rs3) - rs3.log.is_successful = Mock(return_value=True) - er.used_by_components.add(rs1, rs2, rs3) - - er.decontaminate_runcomponents = Mock() - - er.attempt_decontamination(ds1) - rs3.log.is_successful.assert_not_called() - er.decontaminate_runcomponents.assert_not_called() - - def test_attempt_decontamination_last_log_unsuccessful(self): - """ - Attempt bails if the last using component is not successful. - """ - generating_el = ExecLog() - er = ExecRecord(generator=generating_el) - - ds1 = Dataset() - ds2 = Dataset() - ds3 = Dataset() - ero1 = ExecRecordOut(execrecord=er, dataset=ds1) - ero2 = ExecRecordOut(execrecord=er, dataset=ds2) - ero3 = ExecRecordOut(execrecord=er, dataset=ds3) - ero1.is_OK = Mock(return_value=True) - ero2.is_OK = Mock(return_value=True) - ero3.is_OK = Mock(return_value=True) - er.execrecordouts.add(ero1, ero2, ero3) - - rs1 = RunStep(execrecord=er, _runcomponentstate_id=runcomponentstates.QUARANTINED_PK, - end_time=datetime(2000, 2, 14)) - rs1.log = ExecLog(record=rs1) - rs2 = RunStep(execrecord=er, _runcomponentstate_id=runcomponentstates.QUARANTINED_PK, - end_time=datetime(2000, 2, 15)) - rs2.log = ExecLog(record=rs2) - rs3 = RunStep(execrecord=er, _runcomponentstate_id=runcomponentstates.FAILED_PK, - end_time=datetime(2000, 2, 16)) - rs3.log = ExecLog() - rs3.log.is_successful = Mock(return_value=False) - er.used_by_components.add(rs1, rs2, rs3) - - er.decontaminate_runcomponents = Mock() - - er.attempt_decontamination(ds1) - rs3.log.is_successful.assert_called_once_with() - er.decontaminate_runcomponents.assert_not_called() - - -class ExecRecordMockTests(TestCase): - def test_execrecord_input_matches_output_cable_source(self): - trx_out1 = TransformationOutput() - trx_out1.transformationoutput = trx_out1 - execrecord = ExecRecord(generator=ExecLog(record=RunOutputCable( - run=Run(), - pipelineoutputcable=PipelineOutputCable(source=trx_out1)))) - - trx_out2 = TransformationOutput() - trx_out2.transformationoutput = trx_out2 - execrecordin = ExecRecordIn(execrecord=execrecord, dataset=Dataset(id=99), generic_input=trx_out2) - - self.assertRaisesRegexp( - ValidationError, - 'ExecRecordIn "S99" does not denote the TO that feeds the parent ExecRecord POC', - execrecordin.clean) - - def test_execrecord_input_matches_input_cable_source(self): - trx_out1 = TransformationOutput() - trx_out1.transformationoutput = trx_out1 - execrecord = ExecRecord(generator=ExecLog(record=RunSIC( - dest_runstep=RunStep(run=Run()), - PSIC=PipelineStepInputCable(source=trx_out1)))) - - trx_out2 = TransformationOutput() - trx_out2.transformationoutput = trx_out2 - execrecordin = ExecRecordIn(execrecord=execrecord, dataset=Dataset(id=99), generic_input=trx_out2) - - self.assertRaisesRegexp( - ValidationError, - 'ExecRecordIn "S99" does not denote the TO/TI that feeds the parent ExecRecord PSIC', - execrecordin.clean) - - def test_ER_doesnt_link_cable_so_ERI_mustnt_link_TO(self): - method = Method() - method.method = method - execrecord = ExecRecord(generator=ExecLog(record=RunStep(run=Run(), - pipelinestep=PipelineStep(transformation=method)))) - - trx_out = TransformationOutput(dataset_name='ages') - trx_out.transformationoutput = trx_out - execrecordin = ExecRecordIn(execrecord=execrecord, dataset=Dataset(id=99), generic_input=trx_out) - - self.assertRaisesRegexp( - ValidationError, - 'ExecRecordIn "S99=>ages" must refer to a TI of the Method of the parent ExecRecord', - execrecordin.clean) - - def test_general_transf_returns_correct_method(self): - """ - Test if ExecRecord.general_transf() returns the method of the PipelineStep - it was defined with. - """ - method = Method() - method.method = method - execrecord = ExecRecord(generator=ExecLog(record=RunStep(run=Run(), - pipelinestep=PipelineStep(transformation=method)))) - - self.assertEqual(execrecord.general_transf(), method) - - def test_execrecordin_raw_raw(self): - trx_out = TransformationOutput() - trx_out.transformationoutput = trx_out - execrecord = ExecRecord(generator=ExecLog(record=RunOutputCable( - run=Run(), - pipelineoutputcable=PipelineOutputCable(source=trx_out)))) - - execrecordin = ExecRecordIn(execrecord=execrecord, dataset=Dataset(), generic_input=trx_out) - - execrecordin.clean() - - @mocked_relations(Dataset) - def test_execrecordin_raw_cdt(self): - trx_out = TransformationOutput(dataset_idx=3, dataset_name='ages') - trx_out.transformationoutput = trx_out - trx_out.structure = XputStructure() - execrecord = ExecRecord(generator=ExecLog(record=RunOutputCable( - run=Run(), - pipelineoutputcable=PipelineOutputCable(source=trx_out)))) - - execrecordin = ExecRecordIn(execrecord=execrecord, dataset=Dataset(id=99), generic_input=trx_out) - - self.assertRaisesRegexp( - ValidationError, - r'Dataset "S99" \(raw\) cannot feed source "3: ages" \(non-raw\)', - execrecordin.clean) - - def test_execrecordin_cdt_raw(self): - trx_out = TransformationOutput(dataset_idx=3, dataset_name='ages') - trx_out.transformationoutput = trx_out - execrecord = ExecRecord(generator=ExecLog(record=RunOutputCable( - run=Run(), - pipelineoutputcable=PipelineOutputCable(source=trx_out)))) - - dataset = Dataset(id=99) - dataset.structure = DatasetStructure() - execrecordin = ExecRecordIn(execrecord=execrecord, dataset=dataset, generic_input=trx_out) - - self.assertRaisesRegexp( - ValidationError, - r'Dataset "S99" \(non-raw\) cannot feed source "3: ages" \(raw\)', - execrecordin.clean) - - def test_execrecordin_cdt_cdt(self): - trx_out = TransformationOutput(dataset_idx=3, dataset_name='ages') - trx_out.transformationoutput = trx_out - cdt = CompoundDatatype() - trx_out.structure = XputStructure(compounddatatype=cdt) - execrecord = ExecRecord(generator=ExecLog(record=RunOutputCable( - run=Run(), - pipelineoutputcable=PipelineOutputCable(source=trx_out)))) - - dataset = Dataset(id=99) - dataset.structure = DatasetStructure(compounddatatype=cdt) - execrecordin = ExecRecordIn(execrecord=execrecord, dataset=dataset, generic_input=trx_out) - - execrecordin.clean() - - @mocked_relations(CompoundDatatype) - def test_execrecordin_cdts_differ(self): - trx_out = TransformationOutput(dataset_idx=3, dataset_name='ages') - trx_out.transformationoutput = trx_out - cdt1 = CompoundDatatype() - cdt1.members = MockSet(CompoundDatatypeMember(datatype=Datatype()), CompoundDatatypeMember(datatype=Datatype())) - trx_out.structure = XputStructure(compounddatatype=cdt1) - execrecord = ExecRecord(generator=ExecLog(record=RunOutputCable( - run=Run(), - pipelineoutputcable=PipelineOutputCable(source=trx_out)))) - - dataset = Dataset(id=99) - cdt2 = CompoundDatatype() - cdt2.members = MockSet(CompoundDatatypeMember(datatype=Datatype())) - dataset.structure = DatasetStructure(compounddatatype=cdt2) - execrecordin = ExecRecordIn(execrecord=execrecord, dataset=dataset, generic_input=trx_out) - - self.assertRaisesRegexp( - ValidationError, - 'CDT of Dataset "S99" is not a restriction of the required CDT', - execrecordin.clean) - - def test_execrecordin_max_row(self): - trx_out = TransformationOutput(dataset_idx=3, dataset_name='ages') - trx_out.transformationoutput = trx_out - cdt = CompoundDatatype() - trx_out.structure = XputStructure(compounddatatype=cdt, max_row=10) - execrecord = ExecRecord(generator=ExecLog(record=RunOutputCable( - run=Run(), - pipelineoutputcable=PipelineOutputCable(source=trx_out)))) - - dataset = Dataset(id=99) - dataset.structure = DatasetStructure(compounddatatype=cdt, num_rows=10) - execrecordin = ExecRecordIn(execrecord=execrecord, dataset=dataset, generic_input=trx_out) - - execrecordin.clean() - - def test_execrecordin_too_many_rows(self): - trx_out = TransformationOutput(dataset_idx=3, dataset_name='ages') - trx_out.transformationoutput = trx_out - cdt = CompoundDatatype() - trx_out.structure = XputStructure(compounddatatype=cdt, max_row=10) - execrecord = ExecRecord(generator=ExecLog(record=RunOutputCable( - run=Run(), - pipelineoutputcable=PipelineOutputCable(source=trx_out)))) - - dataset = Dataset(id=99) - dataset.structure = DatasetStructure(compounddatatype=cdt, num_rows=11) - execrecordin = ExecRecordIn(execrecord=execrecord, dataset=dataset, generic_input=trx_out) - - self.assertRaisesRegexp( - ValidationError, - 'Dataset "S99" has too many rows to have come from TransformationOutput "3: ages"', - execrecordin.clean) - - def create_pipeline_execrecordout(self): - pipeline = Pipeline() - pipeline.pipeline = pipeline - method = Method() - method.method = method - method_out = TransformationOutput(pk=42, transformation=method) - method_out.transformationoutput = method_out - execrecord = ExecRecord(generator=ExecLog(record=RunOutputCable( - run=Run(), - pipelineoutputcable=PipelineOutputCable(source=method_out, pipeline=pipeline, output_name='ages')))) - pipeline_out = TransformationOutput(pk=43, transformation=pipeline, dataset_name='ages', dataset_idx=3) - pipeline_out.transformationoutput = pipeline_out - dataset = Dataset(id=99) - Dataset.objects = MockSet(dataset) - TransformationXput.objects = MockSet(method_out, pipeline_out) - execrecordout = ExecRecordOut(execrecord=execrecord, dataset=dataset, generic_output=pipeline_out) - return execrecordout - - @mocked_relations(Dataset, TransformationXput) - def test_pipeline_execrecordout_different_pipelines(self): - """ If the parent ER is linked with a POC, the ERO TO must belong to that pipeline """ - - execrecordout = self.create_pipeline_execrecordout() - pipeline2 = Pipeline() - pipeline2.pipeline = pipeline2 - execrecordout.generic_output.transformation = pipeline2 - - self.assertRaisesRegexp( - ValidationError, - 'ExecRecordOut "S99" does not belong to the same pipeline as its parent ExecRecord POC', - execrecordout.clean) - - @mocked_relations(Dataset, TransformationXput) - def test_pipeline_execrecordout_same_pipeline(self): - """ If the parent ER is linked with a POC, the ERO TO must belong to that pipeline """ - - execrecordout = self.create_pipeline_execrecordout() - - execrecordout.clean() - - @mocked_relations(Dataset, TransformationXput) - def test_ER_links_with_POC_and_POC_output_name_must_match_pipeline_TO_name(self): - # The TO must have the same name as the POC which supposedly created it - execrecordout = self.create_pipeline_execrecordout() - execrecordout.execrecord.generator.record.pipelineoutputcable.output_name = 'foo' - - self.assertRaisesRegexp( - ValidationError, - 'ExecRecordOut "S99" does not represent the same output as its parent ExecRecord POC', - execrecordout.clean) - - @mocked_relations(Dataset, TransformationInput, TransformationOutput, TransformationXput) - def test_pipeline_execrecordout_raw_cdt(self): - execrecordout = self.create_pipeline_execrecordout() - execrecordout.generic_output.structure = XputStructure() - self.assertRaisesRegexp( - ValidationError, - r'Dataset "S99" \(raw\) cannot have come from output "3: ages" \(non-raw\)', - execrecordout.clean) - - @mocked_relations(Dataset, TransformationXput) - def test_pipeline_execrecordout_cdt_raw(self): - execrecordout = self.create_pipeline_execrecordout() - execrecordout.dataset.structure = DatasetStructure() - self.assertRaisesRegexp( - ValidationError, - r'Dataset "S99" \(non-raw\) cannot have come from output "3: ages" \(raw\)', - execrecordout.clean) - - @mocked_relations(Dataset, TransformationXput, CompoundDatatype) - def test_pipeline_execrecordout_cdt_cdt01(self): - """NOTE 2018-06-4: this routine was named the same as pne further below, and so - was overwritten. Added the 01 at the end of the name avoid this.""" - cdt = CompoundDatatype() - - execrecordout = self.create_pipeline_execrecordout() - execrecordout.generic_output.structure = XputStructure(compounddatatype=cdt) - execrecordout.dataset.structure = DatasetStructure(compounddatatype=cdt) - execrecordout.clean() - - @mocked_relations(Dataset, TransformationInput, TransformationOutput, TransformationXput, CompoundDatatype) - def test_pipeline_execrecordout_cdt1_cdt2(self): - execrecordout = self.create_pipeline_execrecordout() - cdt1 = CompoundDatatype() - cdt1.members = MockSet(CompoundDatatypeMember(datatype=Datatype())) - execrecordout.generic_output.structure = XputStructure(compounddatatype=cdt1) - cdt2 = CompoundDatatype() - cdt2.members = MockSet(CompoundDatatypeMember(datatype=Datatype()), - CompoundDatatypeMember(datatype=Datatype())) - execrecordout.dataset.structure = DatasetStructure(compounddatatype=cdt2) - self.assertRaisesRegexp( - ValidationError, - 'CDT of Dataset "S99" is not identical to the CDT of the ' - 'TransformationOutput "3: ages" of the generating Pipeline', - execrecordout.clean) - - @mocked_relations(Dataset, TransformationInput, TransformationOutput, TransformationXput, CompoundDatatype) - def test_pipeline_execrecordout_cdt_cdt(self): - cdt = CompoundDatatype() - - execrecordout = self.create_pipeline_execrecordout() - execrecordout.generic_output.structure = XputStructure(compounddatatype=cdt, max_row=10) - execrecordout.dataset.structure = DatasetStructure(compounddatatype=cdt, num_rows=11) - - self.assertRaisesRegexp( - ValidationError, - 'Dataset "S99" was produced by TransformationOutput "3: ages" but has too many rows', - execrecordout.clean) - - def create_method_execrecordout(self): - method = Method() - method.method = method - execrecord = ExecRecord(generator=ExecLog(record=RunStep(run=Run(), - pipelinestep=PipelineStep(transformation=method)))) - - trx_out = TransformationOutput(pk=43, dataset_idx=3, dataset_name='ages') - trx_out.transformationoutput = trx_out - method.outputs = MockSet(trx_out) - dataset = Dataset(id=99) - Dataset.objects = MockSet(dataset) - TransformationXput.objects = MockSet(trx_out) - execrecordout = ExecRecordOut(execrecord=execrecord, dataset=dataset, generic_output=trx_out) - return execrecordout - - @mocked_relations(Dataset, TransformationXput, Transformation) - def test_method_execrecordout_raw_raw(self): - execrecordout = self.create_method_execrecordout() - - execrecordout.clean() - - @mocked_relations(Dataset, - Transformation, - TransformationInput, - TransformationOutput, - TransformationXput, - CompoundDatatype) - def test_method_execrecordout_cdt1_cdt2(self): - execrecordout = self.create_method_execrecordout() - cdt1 = CompoundDatatype() - cdt1.members = MockSet(CompoundDatatypeMember(datatype=Datatype())) - execrecordout.generic_output.structure = XputStructure(compounddatatype=cdt1) - cdt2 = CompoundDatatype() - cdt2.members = MockSet(CompoundDatatypeMember(datatype=Datatype()), - CompoundDatatypeMember(datatype=Datatype())) - execrecordout.dataset.structure = DatasetStructure(compounddatatype=cdt2) - - self.assertRaisesRegexp( - ValidationError, - 'CDT of Dataset "S99" is not the CDT of the TransformationOutput "3: ages" of the generating Method', - execrecordout.clean) - - @mocked_relations(ExecRecord, Dataset, TransformationXput) - def test_cable_execrecordout_raw_raw(self): - execrecordout = self.create_cable_execrecordout() - - execrecordout.clean() - - @mocked_relations(ExecRecord, Dataset, TransformationXput, CompoundDatatype) - def test_cable_execrecordout_cdt1_cdt2(self): - - execrecordout = self.create_cable_execrecordout() - cdt1 = CompoundDatatype() - cdt1.members = MockSet(CompoundDatatypeMember(datatype=Datatype())) - execrecordout.generic_output.structure = XputStructure(compounddatatype=cdt1) - cdt2 = CompoundDatatype() - cdt2.members = MockSet(CompoundDatatypeMember(datatype=Datatype()), - CompoundDatatypeMember(datatype=Datatype())) - execrecordout.dataset.structure = DatasetStructure(compounddatatype=cdt2) - - self.assertRaisesRegexp( - ValidationError, - 'CDT of Dataset "S99" is not a restriction of the CDT of the fed TransformationInput "3: ages"', - execrecordout.clean) - - @mocked_relations(ExecRecord, TransformationInput, TransformationOutput, TransformationXput, Dataset) - def test_trivial_execrecord_matches(self): - """ERs representing trivial PSICs must have the same Dataset on both sides.""" - - execrecord = self.create_cable_execrecordout().execrecord - - execrecord.clean() - - @mocked_relations(ExecRecord, TransformationInput, TransformationOutput, TransformationXput, Dataset) - def test_trivial_execrecord_no_match(self): - """ERs representing trivial PSICs must have the same Dataset on both sides.""" - - execrecord = self.create_cable_execrecordout().execrecord - execrecord.execrecordins.first().dataset = Dataset(id=100) - - self.assertTrue(execrecord.execrecordouts.first().generic_output.is_raw()) - self.assertTrue(execrecord.execrecordins.first().generic_input.is_raw()) - self.assertRaisesRegexp( - ValidationError, - r'ExecRecord "S100 ={2: history:ages\(raw\)}=> S99" represents a trivial cable' - r' but its input and output do not match', - execrecord.clean) - - def create_cable_execrecordout(self): - prev_out = TransformationOutput(pk=42, dataset_idx=1, dataset_name='eras') - prev_out.transformationoutput = prev_out - next_in = TransformationInput(pk=43, dataset_idx=3, dataset_name='ages') - next_in.transformationinput = next_in - execrecord = ExecRecord(generator=ExecLog(record=RunSIC( - dest_runstep=RunStep(run=Run()), - PSIC=PipelineStepInputCable(source=prev_out, dest=next_in, pipelinestep=PipelineStep(name='history', - step_num=2))))) - dataset = Dataset(id=99) - Dataset.objects = MockSet(dataset) - execrecordin = ExecRecordIn(execrecord=execrecord, dataset=dataset, generic_input=prev_out) - TransformationXput.objects = MockSet(next_in) - execrecordout = ExecRecordOut(execrecord=execrecord, dataset=dataset, generic_output=next_in) - execrecord.execrecordins = MockSet(execrecordin) - execrecord.execrecordouts = MockSet(execrecordout) - return execrecordout - - def create_custom_cable_execrecordout(self): - execrecordout = self.create_cable_execrecordout() - execrecord = execrecordout.execrecord - execrecordin = execrecord.execrecordins.first() - string_type = Datatype(name='string') - cdt1 = CompoundDatatype() - field1_in = CompoundDatatypeMember(datatype=string_type, column_idx=1, column_name='age') - field2_in = CompoundDatatypeMember(datatype=string_type, column_idx=2, column_name='start') - cdt1.members = MockSet(field1_in, - field2_in) - execrecordin.generic_input.structure = XputStructure(compounddatatype=cdt1) - execrecordin.dataset.structure = DatasetStructure(compounddatatype=cdt1) - cdt2 = CompoundDatatype() - field1_out = CompoundDatatypeMember(datatype=string_type, column_idx=1, column_name='era') - field2_out = CompoundDatatypeMember(datatype=string_type, column_idx=2, column_name='start') - cdt2.members = MockSet(field1_out, - field2_out) - execrecordout.generic_output.structure = XputStructure(compounddatatype=cdt2) - execrecordout.dataset = Dataset(id=100) - execrecordout.dataset.structure = DatasetStructure(compounddatatype=cdt2) - Dataset.objects = MockSet(execrecordin.dataset, execrecordout.dataset) - cable = execrecord.generator.record.PSIC - cable.custom_wires = MockSet(CustomCableWire(source_pin=field1_in, dest_pin=field1_out), - CustomCableWire(source_pin=field2_in, dest_pin=field2_out)) - return execrecordout - - @mocked_relations(ExecRecord, TransformationXput, Dataset, CompoundDatatype, PipelineCable) - def test_custom_cable(self): - """Test that the Datatypes of Datasets passing through PSICs are properly preserved.""" - execrecordout = self.create_custom_cable_execrecordout() - execrecord = execrecordout.execrecord - execrecord.clean() - - @mocked_relations(ExecRecord, TransformationXput, Dataset, CompoundDatatype, PipelineCable) - def test_custom_cable_mismatch(self): - """Test that the Datatypes of Datasets passing through PSICs are properly preserved.""" - execrecordout = self.create_custom_cable_execrecordout() - execrecord = execrecordout.execrecord - int_type = Datatype(name='int') - execrecordout.dataset.structure.compounddatatype.members.last().datatype = int_type - self.assertRaisesRegexp( - ValidationError, - 'ExecRecord "S99 ={2: history:ages}=> S100" represents a cable, but the Datatype of its destination ' - 'column, "int", does not match the Datatype of its source column, "string"', - execrecord.clean) - - def create_execrecord_with_runstep_states(self, *state_ids): - execrecord = ExecRecord() - for state_id in state_ids: - run_step = RunStep(_runcomponentstate_id=state_id) - run_step.runstep = run_step - execrecord.used_by_components.add(run_step) - return execrecord - - @mocked_relations(ExecRecord) - def test_execrecord_new_never_failed(self): - """An ExecRecord with no RunSteps has never failed.""" - execrecord = self.create_execrecord_with_runstep_states() - - self.assertFalse(execrecord.has_ever_failed()) - - @mocked_relations(ExecRecord) - def test_execrecord_one_good_step(self): - """An ExecRecord with one good RunStep has never failed.""" - execrecord = self.create_execrecord_with_runstep_states(runcomponentstates.SUCCESSFUL_PK) - - self.assertFalse(execrecord.has_ever_failed()) - - @mocked_relations(ExecRecord) - def test_execrecord_two_good_steps(self): - """An ExecRecord with two good RunSteps has never failed.""" - execrecord = self.create_execrecord_with_runstep_states(runcomponentstates.SUCCESSFUL_PK, - runcomponentstates.SUCCESSFUL_PK) - - self.assertFalse(execrecord.has_ever_failed()) - - @mocked_relations(ExecRecord) - def test_execrecord_mixed_steps(self): - """An ExecRecord with two good RunSteps has never failed.""" - execrecord = self.create_execrecord_with_runstep_states(runcomponentstates.SUCCESSFUL_PK, - runcomponentstates.FAILED_PK) - - self.assertTrue(execrecord.has_ever_failed()) diff --git a/kive/librarian/views.py b/kive/librarian/views.py index 58ace5166..a400e477e 100644 --- a/kive/librarian/views.py +++ b/kive/librarian/views.py @@ -90,10 +90,7 @@ def dataset_view(request, dataset_id): # if it was generated, it's anyone who had access to the generating run. addable_users, addable_groups = dataset.other_users_groups() - if dataset.file_source is None: - generating_run = None - else: - generating_run = dataset.file_source.top_level_run + generating_run = None container_dataset = dataset.containers.filter(argument__type='O').first() # Output from which runs? if container_dataset is None: container_run = None diff --git a/kive/metadata/tests.py b/kive/metadata/tests.py index bc8083e8c..b74925717 100644 --- a/kive/metadata/tests.py +++ b/kive/metadata/tests.py @@ -1,2602 +1,14 @@ """ Unit tests for Shipyard metadata models. """ -import os -import re - -from django.core.exceptions import ValidationError -from django.test import TestCase, skipIfDBFeature -from django.core.urlresolvers import reverse, resolve -from django.contrib.auth.models import User, Group - -from rest_framework import status -from rest_framework.test import APIRequestFactory, force_authenticate - -from metadata.models import BasicConstraint, CompoundDatatype, Datatype, kive_user, everyone_group -from librarian.models import Dataset -from metadata.serializers import CompoundDatatypeSerializer -from constants import CDTs, datatypes, groups -import kive.testing_utils as tools -from kive.tests import DuckContext - - -samplecode_path = "../samplecode" - - -@skipIfDBFeature('is_mocked') -class MetadataTestCase(TestCase): - """ - Set up a database state for unit testing. - - Other test classes that require this state can extend this one. - """ - def setUp(self): - tools.create_metadata_test_environment(self) - - def tearDown(self): - tools.clean_up_all_files() - - -class DatatypeTests(MetadataTestCase): - - def setUp(self): - """Add some DTs used to check circular restrictions.""" - MetadataTestCase.setUp(self) - - # Datatypes used to test circular restrictions. - self.dt_1 = Datatype( - name="dt_1", - description="A string (1)", - user=self.myUser) - self.dt_1.save() - self.dt_1.restricts.add(self.string_dt) - - self.dt_2 = Datatype( - name="dt_2", - description="A string (2)", - user=self.myUser) - self.dt_2.save() - self.dt_2.restricts.add(self.string_dt) - - self.dt_3 = Datatype( - name="dt_3", - description="A string (3)", - user=self.myUser) - self.dt_3.save() - self.dt_3.restricts.add(self.string_dt) - - self.dt_4 = Datatype( - name="dt_4", - description="A string (4)", - user=self.myUser) - self.dt_4.save() - self.dt_4.restricts.add(self.string_dt) - - self.dt_5 = Datatype( - name="dt_5", - description="A string (5)", - user=self.myUser) - self.dt_5.save() - self.dt_5.restricts.add(self.string_dt) - - def test_datatype_unicode(self): - """ - Unicode representation must be the instance's name. - - """ - my_datatype = Datatype(name="fhqwhgads", user=self.myUser) - self.assertEqual(unicode(my_datatype), "fhqwhgads") - - # ### Unit tests for datatype.clean (Circular restrictions) ### - - # Direct circular cases: start, middle, end - # Start dt1 restricts dt1, dt3, dt4 - # Middle dt1 restricts dt3, dt1, dt4 - # End dt1 restricts dt3, dt4, dt1 - # Good dt1 restricts dt2, dt3, dt4 - - def test_datatype_circular_direct_start_clean_bad(self): - """ - Circular, direct, start - dt1 restricts dt1, dt3, dt4 - """ - self.dt_1.restricts.add(self.dt_1) - self.dt_1.restricts.add(self.dt_3) - self.dt_1.restricts.add(self.dt_4) - self.dt_1.save() - - self.assertRaisesRegexp(ValidationError, - re.escape('Datatype "{}" has a circular restriction'.format(self.dt_1)), - self.dt_1.clean) - - def test_datatype_circular_direct_middle_clean_bad(self): - """ - Circular, direct, middle - dt1 restricts dt3, dt1, dt4 - """ - self.dt_1.restricts.add(self.dt_3) - self.dt_1.restricts.add(self.dt_1) - self.dt_1.restricts.add(self.dt_4) - self.dt_1.save() - - self.assertRaisesRegexp(ValidationError, - re.escape('Datatype "{}" has a circular restriction'.format(self.dt_1)), - self.dt_1.clean) - - def test_datatype_circular_direct_end_clean_bad(self): - """ - Circular, direct, middle - dt1 restricts dt3, dt4, dt1 - """ - self.dt_1.restricts.add(self.dt_3) - self.dt_1.restricts.add(self.dt_4) - self.dt_1.restricts.add(self.dt_1) - self.dt_1.save() - - self.assertRaisesRegexp(ValidationError, - re.escape('Datatype "{}" has a circular restriction'.format(self.dt_1)), - self.dt_1.clean) - - def test_datatype_circular_direct_clean_good(self): - """ - dt1 restricts dt2, dt3, dt4 - """ - self.dt_1.restricts.add(self.dt_2) - self.dt_1.restricts.add(self.dt_3) - self.dt_1.restricts.add(self.dt_4) - self.dt_1.save() - self.assertEqual(self.dt_1.clean(), None) - - def test_datatype_circular_recursive_begin_clean_bad(self): - """ - dt1 restricts dt2, dt3, dt4 - dt2 restricts dt1 - """ - self.dt_1.restricts.add(self.dt_2) - self.dt_1.restricts.add(self.dt_3) - self.dt_1.restricts.add(self.dt_4) - self.dt_1.save() - - self.dt_2.restricts.add(self.dt_1) - self.dt_2.save() - - self.assertRaisesRegexp(ValidationError, - re.escape('Datatype "{}" has a circular restriction'.format(self.dt_1)), - self.dt_1.clean) - - def test_datatype_circular_recursive_middle_clean_bad(self): - """ - dt1 restricts dt2, dt3, dt4 - dt3 restricts dt1 - """ - self.dt_1.restricts.add(self.dt_2) - self.dt_1.restricts.add(self.dt_3) - self.dt_1.restricts.add(self.dt_4) - self.dt_1.save() - - self.dt_3.restricts.add(self.dt_1) - self.dt_3.save() - - self.assertRaisesRegexp(ValidationError, - re.escape('Datatype "{}" has a circular restriction'.format(self.dt_1)), - self.dt_1.clean) - - def test_datatype_circular_recursive_end_clean_bad(self): - """ - dt1 restricts dt2, dt3, dt4 - dt4 restricts dt1 - """ - self.dt_1.restricts.add(self.dt_2) - self.dt_1.restricts.add(self.dt_3) - self.dt_1.restricts.add(self.dt_4) - self.dt_1.save() - self.dt_4.restricts.add(self.dt_1) - self.dt_4.save() - - self.assertRaisesRegexp(ValidationError, - re.escape('Datatype "{}" has a circular restriction'.format(self.dt_1)), - self.dt_1.clean) - - def test_datatype_circular_recursive_clean_good1(self): - """ - dt1 restricts dt2, dt3, dt4 - dt2 restricts dt5 - """ - self.dt_1.restricts.add(self.dt_2) - self.dt_1.restricts.add(self.dt_3) - self.dt_1.restricts.add(self.dt_4) - self.dt_1.save() - self.dt_2.restricts.add(self.dt_5) - self.dt_2.save() - self.assertEqual(self.dt_1.clean(), None) - - def test_datatype_circular_recursive_clean_good2(self): - """ - dt1 restricts dt2, dt3, dt4 - dt3 restricts dt5 - """ - self.dt_1.restricts.add(self.dt_2) - self.dt_1.restricts.add(self.dt_3) - self.dt_1.restricts.add(self.dt_4) - self.dt_1.save() - self.dt_3.restricts.add(self.dt_5) - self.dt_3.save() - self.assertEqual(self.dt_1.clean(), None) - - def test_datatype_circular_recursive_clean_good3(self): - """ - dt1 restricts dt2, dt3, dt4 - dt4 restricts dt5 - """ - self.dt_1.restricts.add(self.dt_2) - self.dt_1.restricts.add(self.dt_3) - self.dt_1.restricts.add(self.dt_4) - self.dt_1.save() - self.dt_4.restricts.add(self.dt_5) - self.dt_4.save() - self.assertEqual(self.dt_1.clean(), None) - - def test_datatype_circular_recursive_clean_good4(self): - """ - dt1 restricts dt2, dt3, dt4 - dt2 restricts dt4 - """ - self.dt_1.restricts.add(self.dt_2) - self.dt_1.restricts.add(self.dt_3) - self.dt_1.restricts.add(self.dt_4) - self.dt_1.save() - self.dt_2.restricts.add(self.dt_4) - self.dt_2.save() - self.assertEqual(self.dt_1.clean(), None) - - def test_datatype_circular_recursive_clean_good5(self): - """ - dt1 restricts dt2, dt3, dt4 - dt3 restricts dt4 - """ - self.dt_1.restricts.add(self.dt_2) - self.dt_1.restricts.add(self.dt_3) - self.dt_1.restricts.add(self.dt_4) - self.dt_1.save() - self.dt_3.restricts.add(self.dt_4) - self.dt_3.save() - self.assertEqual(self.dt_1.clean(), None) - - def test_datatype_circular_recursive_clean_good6(self): - """ - dt1 restricts dt2, dt3, dt4 - dt4 restricts dt2 - """ - self.dt_1.restricts.add(self.dt_2) - self.dt_1.restricts.add(self.dt_3) - self.dt_1.restricts.add(self.dt_4) - self.dt_1.save() - self.dt_4.restricts.add(self.dt_2) - self.dt_4.save() - self.assertEqual(self.dt_1.clean(), None) - - def test_datatype_direct_is_restricted_by_1(self): - """ - dt1 restricts dt2 - dt1.is_restricted_by(dt2) - FALSE - dt2.is_restricted_by(dt1) - TRUE - """ - self.dt_1.restricts.add(self.dt_2) - self.dt_1.save() - - self.assertEqual(self.dt_1.is_restricted_by(self.dt_2), False) - self.assertEqual(self.dt_2.is_restricted_by(self.dt_1), True) - - def test_datatype_direct_is_restricted_by_2(self): - """ - dt1 and dt2 exist but do not restrict each other - dt1.is_restricted_by(dt2) - FALSE - dt2.is_restricted_by(dt1) - FALSE - """ - self.assertEqual(self.dt_1.is_restricted_by(self.dt_2), False) - self.assertEqual(self.dt_2.is_restricted_by(self.dt_1), False) - - def test_datatype_recursive_is_restricted_by_1(self): - """ - dt1 restricts dt2, dt2 restricts dt3 - - dt1.is_restricted_by(dt3) - FALSE - dt3.is_restricted_by(dt1) - TRUE - dt1.is_restricted_by(dt2) - FALSE - dt2.is_restricted_by(dt1) - TRUE - """ - - self.dt_1.restricts.add(self.dt_2) - self.dt_1.save() - self.dt_2.restricts.add(self.dt_3) - self.dt_2.save() - - self.assertEqual(self.dt_1.is_restricted_by(self.dt_3), False) - self.assertEqual(self.dt_3.is_restricted_by(self.dt_1), True) - self.assertEqual(self.dt_1.is_restricted_by(self.dt_2), False) - self.assertEqual(self.dt_2.is_restricted_by(self.dt_1), True) - - def test_datatype_recursive_is_restricted_by_2(self): - """ - dt1 restricts dt[2,3,4] - dt2 restricts dt5 - """ - - self.dt_1.restricts.add(self.dt_2) - self.dt_1.restricts.add(self.dt_3) - self.dt_1.restricts.add(self.dt_4) - self.dt_1.save() - self.dt_2.restricts.add(self.dt_5) - self.dt_2.save() - self.assertEqual(self.dt_1.is_restricted_by(self.dt_2), False) - self.assertEqual(self.dt_1.is_restricted_by(self.dt_3), False) - self.assertEqual(self.dt_1.is_restricted_by(self.dt_4), False) - self.assertEqual(self.dt_1.is_restricted_by(self.dt_5), False) - self.assertEqual(self.dt_5.is_restricted_by(self.dt_1), True) - - def test_datatype_recursive_is_restricted_by_3(self): - """ - dt1 restricts dt[2,3,4] - dt3 restricts dt5 - """ - - self.dt_1.restricts.add(self.dt_2) - self.dt_1.restricts.add(self.dt_3) - self.dt_1.restricts.add(self.dt_4) - self.dt_1.save() - self.dt_3.restricts.add(self.dt_5) - self.dt_3.save() - self.assertEqual(self.dt_1.is_restricted_by(self.dt_2), False) - self.assertEqual(self.dt_1.is_restricted_by(self.dt_3), False) - self.assertEqual(self.dt_1.is_restricted_by(self.dt_4), False) - self.assertEqual(self.dt_1.is_restricted_by(self.dt_5), False) - self.assertEqual(self.dt_5.is_restricted_by(self.dt_1), True) - - def test_datatype_recursive_is_restricted_by_4(self): - """ - dt1 restricts dt[2,3,4] - dt4 restricts dt5 - """ - - self.dt_1.restricts.add(self.dt_2) - self.dt_1.restricts.add(self.dt_3) - self.dt_1.restricts.add(self.dt_4) - self.dt_1.save() - self.dt_4.restricts.add(self.dt_5) - self.dt_4.save() - self.assertEqual(self.dt_1.is_restricted_by(self.dt_2), False) - self.assertEqual(self.dt_1.is_restricted_by(self.dt_3), False) - self.assertEqual(self.dt_1.is_restricted_by(self.dt_4), False) - self.assertEqual(self.dt_1.is_restricted_by(self.dt_5), False) - self.assertEqual(self.dt_5.is_restricted_by(self.dt_1), True) - - def test_datatype_no_restriction_clean_good(self): - """ - Datatype without any restrictions. - """ - self.assertEqual(self.dt_1.clean(), None) - - def test_datatype_nested_valid_restrictions_clean_good(self): - """ - Datatypes such that A restricts B, and B restricts C - """ - self.dt_1.restricts.add(self.dt_2) - self.dt_1.save() - self.dt_2.restricts.add(self.dt_3) - self.dt_2.save() - self.assertEqual(self.dt_1.clean(), None) - self.assertEqual(self.dt_2.clean(), None) - self.assertEqual(self.dt_3.clean(), None) - - def test_datatype_nested_invalid_restrictions_scrambled_clean_bad(self): - """ - Datatypes are restricted to constrain execution order such that: - A restricts C - A restricts B - B restricts C - C restricts A - """ - - self.dt_1.restricts.add(self.dt_3) - self.dt_1.save() - self.dt_1.restricts.add(self.dt_2) - self.dt_1.save() - self.dt_2.restricts.add(self.dt_3) - self.dt_2.save() - self.dt_3.restricts.add(self.dt_1) - self.dt_3.save() - - self.assertRaisesRegexp(ValidationError, - re.escape('Datatype "{}" has a circular restriction'.format(self.dt_1)), - self.dt_1.clean) - - def test_datatype_direct_circular_restriction_clean_bad(self): - """ - Datatype directly restricts itself: A restricts A - """ - - self.dt_1.restricts.add(self.dt_1) - self.dt_1.save() - - self.assertRaisesRegexp(ValidationError, - re.escape('Datatype "{}" has a circular restriction'.format(self.dt_1)), - self.dt_1.clean) - - def test_datatype_circular_restriction_indirect_clean(self): - """ - Datatype restricts itself through intermediary: - A restricts B - B restricts A - """ - - self.dt_1.restricts.add(self.dt_2) - self.dt_1.save() - self.dt_2.restricts.add(self.dt_1) - self.dt_2.save() - - self.assertRaisesRegexp(ValidationError, - re.escape('Datatype "{}" has a circular restriction'.format(self.dt_1)), - self.dt_1.clean) - - def test_datatype_clean_no_restricts(self): - """ - Clean on a Datatype with no restrictions should pass. - """ - datatype = Datatype( - name="squeaky", - description="a clean, new datatype", - user=self.myUser) - # Note that this passes if the next line is uncommented. - # datatype.save() - self.assertEqual(datatype.clean(), None) - - ######## - # New tests to check the new functionality in Datatype.clean() - # that checks BasicConstraints, the prototype Dataset, etc. - - def __test_clean_restrict_same_builtin_multiply_good_h(self, builtin_type): - """ - Helper for testing clean() on cases where a Datatype restricts several supertypes with the same builtin type. - """ - super_DT = Datatype(name="SuperDT", description="Supertype 1", user=self.myUser) - super_DT.full_clean() - super_DT.save() - super_DT.restricts.add(builtin_type) - - super2_DT = Datatype(name="SuperDT2", description="Supertype 2", user=self.myUser) - super2_DT.full_clean() - super2_DT.save() - super2_DT.restricts.add(builtin_type) - - my_DT = Datatype(name="MyDT", description="Datatype with two built-in supertypes", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(builtin_type, builtin_type) - - your_DT = Datatype(name="YourDT", description="Datatype with two supertypes", user=self.myUser) - your_DT.full_clean() - your_DT.save() - your_DT.restricts.add(super_DT, super2_DT) - - self.assertEquals(my_DT.clean(), None) - self.assertEquals(your_DT.clean(), None) - - def test_clean_restrict_several_str_good(self): - """ - Testing clean() on the case where a Datatype restricts several string supertypes. - """ - self.__test_clean_restrict_same_builtin_multiply_good_h(self.STR) - - def test_clean_restrict_several_int_good(self): - """ - Testing clean() on the case where a Datatype restricts several integer supertypes. - """ - self.__test_clean_restrict_same_builtin_multiply_good_h(self.INT) - - def test_clean_restrict_several_float_good(self): - """ - Testing clean() on the case where a Datatype restricts several float supertypes. - """ - self.__test_clean_restrict_same_builtin_multiply_good_h(self.FLOAT) - - def test_clean_restrict_several_bool_good(self): - """ - Testing clean() on the case where a Datatype restricts several Boolean supertypes. - """ - self.__test_clean_restrict_same_builtin_multiply_good_h(self.BOOL) - - def test_clean_restrict_int_float_good(self): - """ - Testing clean() on the case where a Datatype restricts both integer and float supertypes. - """ - super_DT = Datatype(name="SuperDT", description="Supertype 1", user=self.myUser) - super_DT.full_clean() - super_DT.save() - super_DT.restricts.add(self.INT) - - super2_DT = Datatype(name="SuperDT2", description="Supertype 2", user=self.myUser) - super2_DT.full_clean() - super2_DT.save() - super2_DT.restricts.add(self.FLOAT) - - my_DT = Datatype(name="MyDT", description="Datatype with two built-in supertypes", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(self.INT, self.FLOAT) - - your_DT = Datatype(name="YourDT", description="Datatype with two supertypes", user=self.myUser) - your_DT.full_clean() - your_DT.save() - your_DT.restricts.add(super_DT, super2_DT) - - self.assertEquals(my_DT.clean(), None) - self.assertEquals(your_DT.clean(), None) - - #### - def __test_clean_restrict_several_builtins_bad_h(self, builtin_type_1, builtin_type_2): - """ - Helper for testing clean() on cases where a Datatype restricts supertypes with non-compatible builtin types. - """ - super_DT = Datatype(name="SuperDT", description="Supertype 1", user=self.myUser) - super_DT.full_clean() - super_DT.save() - super_DT.restricts.add(builtin_type_1) - - super2_DT = Datatype(name="SuperDT2", description="Supertype 2", user=self.myUser) - super2_DT.full_clean() - super2_DT.save() - super2_DT.restricts.add(builtin_type_2) - - my_DT = Datatype(name="MyDT", description="Datatype with two built-in supertypes", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(builtin_type_1, builtin_type_2) - - your_DT = Datatype(name="YourDT", description="Datatype with two supertypes", user=self.myUser) - your_DT.full_clean() - your_DT.save() - your_DT.restricts.add(super_DT, super2_DT) - - self.assertRaisesRegexp(ValidationError, - re.escape(('Datatype "{}" restricts multiple built-in, non-numeric types' - .format(my_DT))), - my_DT.clean) - - self.assertRaisesRegexp(ValidationError, - re.escape(('Datatype "{}" restricts multiple built-in, non-numeric types' - .format(your_DT))), - your_DT.clean) - - def test_clean_restricts_str_int_bad(self): - """ - Tests clean() on the case of a Datatype restricting both STR and INT. - """ - self.__test_clean_restrict_several_builtins_bad_h(self.STR, self.INT) - - def test_clean_restricts_str_float_bad(self): - """ - Tests clean() on the case of a Datatype restricting both STR and FLOAT. - """ - self.__test_clean_restrict_several_builtins_bad_h(self.STR, self.FLOAT) - - def test_clean_restricts_str_bool_bad(self): - """ - Tests clean() on the case of a Datatype restricting both STR and BOOL. - """ - self.__test_clean_restrict_several_builtins_bad_h(self.STR, self.BOOL) - - def test_clean_restricts_float_bool_bad(self): - """ - Tests clean() on the case of a Datatype restricting both FLOAT and BOOL. - """ - self.__test_clean_restrict_several_builtins_bad_h(self.FLOAT, self.BOOL) - - def test_clean_restricts_int_bool_bad(self): - """ - Tests clean() on the case of a Datatype restricting both INT and BOOL. - """ - self.__test_clean_restrict_several_builtins_bad_h(self.BOOL, self.INT) - - #### - def test_clean_prototype_good(self): - """ - Testing clean() on a Datatype whose prototype is well-defined. - """ - # Make a Dataset for the prototype CSV file. - PROTOTYPE_CDT = CompoundDatatype.objects.get(pk=CDTs.PROTOTYPE_PK) - DNA_prototype = Dataset.create_dataset(os.path.join(samplecode_path, "DNAprototype.csv"), - user=self.myUser, cdt=PROTOTYPE_CDT, - name="DNAPrototype", description="Prototype for the DNA Datatype") - - self.DNA_dt.prototype = DNA_prototype - - self.assertEquals(self.DNA_dt.clean(), None) - - def test_clean_raw_prototype_bad(self): - """ - Testing clean() on a Datatype whose prototype is raw. - """ - DNA_raw_prototype = Dataset.create_dataset(os.path.join(samplecode_path, "DNAprototype.csv"), - user=self.myUser, cdt=None, - name="RawPrototype", description="Prototype that is raw") - - self.DNA_dt.prototype = DNA_raw_prototype - PROTOTYPE_CDT = CompoundDatatype.objects.get(pk=CDTs.PROTOTYPE_PK) - self.assertRaisesRegexp(ValidationError, - re.escape('Prototype Dataset for Datatype "{}" should have CompoundDatatype "{}", ' - 'but it is raw'.format(self.DNA_dt, PROTOTYPE_CDT)), - self.DNA_dt.clean) - - def test_clean_prototype_wrong_CDT_bad(self): - """ - Testing clean() on a Datatype whose prototype has the incorrect CDT. - """ - wrong_CDT = CompoundDatatype(user=self.myUser) - wrong_CDT.save() - wrong_CDT.members.create(datatype=self.STR, column_name="example", column_idx=1, - blankable=True) - wrong_CDT.members.create(datatype=self.BOOL, column_name="thisshouldbesomethingelse", column_idx=2) - wrong_CDT.clean() - - DNA_prototype_bad_CDT = Dataset.create_dataset(os.path.join(samplecode_path, "DNAprototype_bad_CDT.csv"), - user=self.myUser, cdt=wrong_CDT, - name="BadCDTPrototype", - description="Prototype with a bad CDT") - - self.DNA_dt.prototype = DNA_prototype_bad_CDT - - PROTOTYPE_CDT = CompoundDatatype.objects.get(pk=CDTs.PROTOTYPE_PK) - self.assertRaisesRegexp(ValidationError, - re.escape(('Prototype Dataset for Datatype "{}" should have CompoundDatatype "{}", ' - 'but it has "{}"'.format(self.DNA_dt, PROTOTYPE_CDT, wrong_CDT))), - self.DNA_dt.clean) - - # Propagation of BasicConstraint errors is checked thoroughly in the BasicConstraint - # tests. Let's just quickly check two cases. - def test_clean_BC_clean_propagation_good(self): - """ - Testing to confirm that BasicConstraint.clean() is called from Datatype.clean(): good case. - """ - constr_DT = Datatype(name="ConstrainedDatatype", description="Datatype with good BasicConstraint", - user=self.myUser) - constr_DT.full_clean() - constr_DT.save() - constr_DT.restricts.add(self.FLOAT) - - constr_DT.basic_constraints.create(ruletype=BasicConstraint.MIN_VAL, rule="7") - - self.assertEquals(constr_DT.clean(), None) - - def test_clean_BC_clean_propagation_bad(self): - """ - Testing to confirm that BasicConstraint.clean() is called from Datatype.clean(): bad case. - """ - constr_DT = Datatype(name="BadlyConstrainedDatatype", description="Datatype with bad BasicConstraint", - user=self.myUser) - constr_DT.full_clean() - constr_DT.save() - constr_DT.restricts.add(self.FLOAT) - - constr = constr_DT.basic_constraints.create(ruletype=BasicConstraint.DATETIMEFORMAT, rule="%Y %b %d") - - self.assertRaisesRegexp(ValidationError, - re.escape((('BasicConstraint "{}" specifies a date/time format, but its parent ' - 'Datatype "{}" has builtin type "{}"').format( - constr, - constr_DT, - self.FLOAT))), - constr_DT.clean) - - # Cases where a Datatype has a good BasicConstraint associated to it are well-tested in the - # BasicConstraint tests. Again we quickly check a couple of cases. - def test_clean_has_good_regexp_good(self): - """ - Testing clean() on a Datatype with a good REGEXP attached. - """ - constr_DT = Datatype(name="ConstrainedDatatype", description="Datatype with good REGEXP", - user=self.myUser) - constr_DT.full_clean() - constr_DT.save() - constr_DT.restricts.add(self.FLOAT) - - constr_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule=".*") - - self.assertEquals(constr_DT.clean(), None) - - def test_clean_has_good_min_val_good(self): - """ - Testing clean() on a Datatype with a good MIN_VAL attached. - """ - constr_DT = Datatype(name="ConstrainedDatatype", description="Datatype with good MIN_VAL", - user=self.myUser) - constr_DT.full_clean() - constr_DT.save() - constr_DT.restricts.add(self.INT) - - constr_DT.basic_constraints.create(ruletype=BasicConstraint.MIN_VAL, rule="-22.3") - - self.assertEquals(constr_DT.clean(), None) - - # Cases where a Datatype has several good BCs attached. - def test_clean_float_has_several_good_BCs_good(self): - """ - Testing clean() on a Datatype with several good BCs attached. - """ - constr_DT = Datatype(name="ConstrainedDatatype", description="FLOAT with good BCs", - user=self.myUser) - constr_DT.full_clean() - constr_DT.save() - constr_DT.restricts.add(self.FLOAT) - - constr_DT.basic_constraints.create(ruletype=BasicConstraint.MAX_VAL, rule="1000") - constr_DT.basic_constraints.create(ruletype=BasicConstraint.MIN_VAL, rule="1.7") - constr_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule="[1-9]*") - - self.assertEquals(constr_DT.clean(), None) - - def test_clean_string_has_several_good_BCs_good(self): - """ - Testing clean() on a string Datatype with several good BCs attached. - """ - constr_DT = Datatype(name="ConstrainedDatatype", description="STR with good BCs", - user=self.myUser) - constr_DT.full_clean() - constr_DT.save() - constr_DT.restricts.add(self.STR) - - constr_DT.basic_constraints.create(ruletype=BasicConstraint.MIN_LENGTH, rule="6") - constr_DT.basic_constraints.create(ruletype=BasicConstraint.DATETIMEFORMAT, rule="%Y %b %d") - - self.assertEquals(constr_DT.clean(), None) - - #### - # Cases where a Datatype has multiple BasicConstraints of the same type. - def __test_clean_multiple_same_BCs_bad_h(self, builtin_type, rules, multiple_BC_type): - """ - Helper for the case where a Datatype has multiple BasicConstraints of the same type. - - rules is a list of tuples of the form (ruletype, rule). - multiple_BC_type is one of BasicConstraint.(MIN|MAX)_(LENGTH|VAL) or - BasicConstraint.DATETIMEFORMAT. - """ - constr_DT = Datatype(name="MultiplyConstrainedDatatype", - description="Datatype with several BCs of the same type", - user=self.myUser) - constr_DT.full_clean() - constr_DT.save() - constr_DT.restricts.add(builtin_type) - - counts = {} - bad_ruletypes = set() - for curr_ruletype, curr_rule in rules: - try: - counts[curr_ruletype] += 1 - bad_ruletypes.add(curr_ruletype) - except KeyError: - counts[curr_ruletype] = 1 - constr_DT.basic_constraints.create(ruletype=curr_ruletype, rule="{}".format(curr_rule)) - - possible_matches = [re.escape('Datatype "{}" has {} constraints of type {}, but should have at most one'. - format(constr_DT, counts[x], x)) - for x in bad_ruletypes] - match_pattern = "|".join(possible_matches) - - self.assertRaisesRegexp(ValidationError, match_pattern, constr_DT.clean) - - def test_clean_int_multiple_min_val_bad(self): - """ - Testing clean() on an integer Datatype with multiple MIN_VAL restrictions. - """ - self.__test_clean_multiple_same_BCs_bad_h( - self.INT, - [(BasicConstraint.MIN_VAL, 6), (BasicConstraint.MIN_VAL, 8)], - BasicConstraint.MIN_VAL - ) - - def test_clean_float_multiple_max_val_bad(self): - """ - Testing clean() on a float Datatype with multiple MAX_VAL restrictions. - """ - self.__test_clean_multiple_same_BCs_bad_h( - self.FLOAT, - [(BasicConstraint.MAX_VAL, 1220), (BasicConstraint.MAX_VAL, 6)], - BasicConstraint.MAX_VAL - ) - - def test_clean_str_multiple_min_length_bad(self): - """ - Testing clean() on a string Datatype with multiple MIN_LENGTH restrictions. - """ - self.__test_clean_multiple_same_BCs_bad_h( - self.STR, - [(BasicConstraint.MIN_LENGTH, 1), (BasicConstraint.MIN_LENGTH, 2)], - BasicConstraint.MIN_LENGTH - ) - - def test_clean_str_multiple_max_length_bad(self): - """ - Testing clean() on a string Datatype with multiple MAX_LENGTH restrictions. - """ - self.__test_clean_multiple_same_BCs_bad_h( - self.STR, - [(BasicConstraint.MAX_LENGTH, 7), (BasicConstraint.MAX_LENGTH, 4), (BasicConstraint.MAX_LENGTH, 7)], - BasicConstraint.MAX_LENGTH - ) - - def test_clean_str_multiple_dtf_bad(self): - """ - Testing clean() on a string Datatype with multiple DATETIMEFORMAT restrictions. - """ - self.__test_clean_multiple_same_BCs_bad_h( - self.STR, - [(BasicConstraint.DATETIMEFORMAT, "%Y"), (BasicConstraint.DATETIMEFORMAT, "foo"), - (BasicConstraint.MAX_LENGTH, 7)], - BasicConstraint.DATETIMEFORMAT - ) - - def test_clean_float_some_good_some_multiple_BCs_bad(self): - """ - Testing clean() on a float Datatype with several BCs but with at least one multiply-defined. - """ - # Note that here, both MIN_VAL and MAX_VAL are multiply-defined, - # so either one could fail. That's why we pass a regexp for - # multiple_BC_type. - self.__test_clean_multiple_same_BCs_bad_h( - self.FLOAT, - [(BasicConstraint.MIN_VAL, "7"), (BasicConstraint.MAX_VAL, "15"), - (BasicConstraint.MIN_VAL, "13"), (BasicConstraint.REGEXP, "[1-9]+"), - (BasicConstraint.MAX_VAL, "19")], - "(?:{}|{})".format(BasicConstraint.MIN_VAL, BasicConstraint.MAX_VAL) - ) - - def _setup_datatype(self, name, desc, rules, restricts): - """ - Helper function to create a Datatype. Rules is a list of tuples (ruletype, rule), - and restricts is a list of supertypes. - """ - dt = Datatype(name=name, description=desc, user=self.myUser) - dt.full_clean() - dt.save() - for supertype in restricts: - dt.restricts.add(supertype) - for ruletype, rule in rules: - if ruletype: - dt.basic_constraints.create(ruletype=ruletype, rule=rule) - return dt - - def _setup_inheriting_datatype(self, - super_name, super_desc, super_ruletype, super_rule, super_builtin, - cnstr_name, cnstr_desc, cnstr_ruletype, cnstr_rule): - """ - Helper function to create a pair of Datatypes, one inheriting - from the other. - """ - super_DT = self._setup_datatype(super_name, super_desc, [(super_ruletype, super_rule)], [super_builtin]) - constr_DT = self._setup_datatype(cnstr_name, cnstr_desc, [(cnstr_ruletype, cnstr_rule)], [super_DT]) - return (super_DT, constr_DT) - - def _setup_inheriting_datatype2(self, - super1_name, super1_desc, super1_ruletype, super1_rule, super1_builtin, - super2_name, super2_desc, super2_ruletype, super2_rule, super2_builtin, - constr_name, constr_desc, constr_ruletype, constr_rule): - """ - Helper function to create three Datatypes, the first two being - supertypes of the third. - """ - super1_DT, constr_DT = self._setup_inheriting_datatype( - super1_name, super1_desc, super1_ruletype, super1_rule, super1_builtin, - constr_name, constr_desc, constr_ruletype, constr_rule) - super2_DT = self._setup_datatype(super2_name, super2_desc, [(super2_ruletype, super2_rule)], [super2_builtin]) - constr_DT.restricts.add(super2_DT) - return (super1_DT, super2_DT, constr_DT) - - #### - def __test_clean_num_constraint_conflicts_with_supertypes_h(self, builtin_type, BC_type, constr_val, - supertype_constr_val): - """ - Helper to test cases where numerical constraints conflict with those of the supertypes. - """ - super_DT, constr_DT = self._setup_inheriting_datatype( - "ParentDT", - "Parent with constraint", - BC_type, - supertype_constr_val, - builtin_type, - "ConstrDT", - "Datatype whose constraint conflicts with parent", - BC_type, - constr_val) - - if BC_type == BasicConstraint.MIN_LENGTH: - error_msg = 'Datatype "{}" has MIN_LENGTH {}, but its supertype "{}" has a longer or equal MIN_LENGTH of {}' - elif BC_type == BasicConstraint.MAX_LENGTH: - error_msg = ('Datatype "{}" has MAX_LENGTH {}, but its supertype ' - '"{}" has a shorter or equal MAX_LENGTH of {}') - elif BC_type == BasicConstraint.MIN_VAL: - error_msg = 'Datatype "{}" has MIN_VAL {}, but its supertype "{}" has a larger or equal MIN_VAL of {}' - elif BC_type == BasicConstraint.MAX_VAL: - error_msg = 'Datatype "{}" has MAX_VAL {}, but its supertype "{}" has a smaller or equal MAX_VAL of {}' - - self.assertRaisesRegexp(ValidationError, - re.escape(error_msg.format(constr_DT, constr_val, super_DT, supertype_constr_val)), - constr_DT.clean) - - def test_clean_int_min_val_supertype_conflict_bad(self): - """ - Testing clean() on an integer whose MIN_VAL conflicts with its supertypes'. - """ - self.__test_clean_num_constraint_conflicts_with_supertypes_h( - self.INT, - BasicConstraint.MIN_VAL, - 7, 9 - ) - - def test_clean_float_max_val_supertype_conflict_bad(self): - """ - Testing clean() on an integer whose MIN_VAL conflicts with its supertypes'. - """ - self.__test_clean_num_constraint_conflicts_with_supertypes_h( - self.FLOAT, - BasicConstraint.MAX_VAL, - 11, 10.7 - ) - - def test_clean_str_min_length_supertype_conflict_bad(self): - """ - Testing clean() on an integer whose MIN_VAL conflicts with its supertypes'. - """ - self.__test_clean_num_constraint_conflicts_with_supertypes_h( - self.STR, - BasicConstraint.MIN_LENGTH, - 9, 10 - ) - - def test_clean_str_max_length_supertype_conflict_bad(self): - """ - Testing clean() on an integer whose MAX_VAL conflicts with its supertypes'. - """ - self.__test_clean_num_constraint_conflicts_with_supertypes_h( - self.STR, - BasicConstraint.MAX_LENGTH, - 223, 20 - ) - - def test_clean_dtf_conflict_with_supertype_bad(self): - """ - Testing clean() on the case where a Datatype has a DATETIMEFORMAT but so does its supertype. - """ - _super_DT, constr_DT = self._setup_inheriting_datatype( - "DateTimeDT", - "String with a DATETIMEFORMAT", - BasicConstraint.DATETIMEFORMAT, - "%Y %b %d", - self.STR, - "OverwritingDateTimeDT", - "String with a DATETIMEFORMAT whose parent also has one", - BasicConstraint.DATETIMEFORMAT, - "%Y-%b-%d") - - self.assertRaisesRegexp(ValidationError, - re.escape(('Datatype "{}" should have only one DATETIMEFORMAT restriction acting on ' - 'it, but it has {}'.format(constr_DT, 2))), - constr_DT.clean) - - def test_clean_several_supertypes_have_dtfs_bad(self): - """ - Testing clean() on the case where a Datatype has several supertypes with DATETIMEFORMATs. - """ - dtf = BasicConstraint.DATETIMEFORMAT - _super_DT, _second_DT, constr_DT = self._setup_inheriting_datatype2( - "DateTimeDT", "String with a DATETIMEFORMAT", dtf, "%Y %b %d", self.STR, - "OverwritingDateTimeDT", "Second string with a DATETIMEFORMAT", dtf, "%Y %b %d", self.STR, - "OverwritingDateTimeChildDT", "String with a DATETIMEFORMAT whose parent also has one", dtf, "%Y %b %d") - - self.assertRaisesRegexp(ValidationError, - re.escape(('Datatype "{}" should have only one DATETIMEFORMAT restriction acting on ' - 'it, but it has {}'.format(constr_DT, 3))), - constr_DT.clean) - - def test_clean_dtf_several_supertypes_one_has_dtf_bad(self): - """ - Testing clean() on the case where a Datatype has a DATETIMEFORMAT and several supertypes, one which has one. - """ - dtf = BasicConstraint.DATETIMEFORMAT - _super_DT, _second_DT, constr_DT = self._setup_inheriting_datatype2( - "DateTimeDT", "String with a DATETIMEFORMAT", dtf, "%Y %b %d", self.STR, - "OtherDT", "String by a different name", None, None, self.STR, - "OverwritingDateTimeDT", "String with a DATETIMEFORMAT whose parent also has one", dtf, "%Y %d") - - self.assertRaisesRegexp(ValidationError, - re.escape(('Datatype "{}" should have only one DATETIMEFORMAT restriction acting on ' - 'it, but it has {}'.format(constr_DT, 2))), - constr_DT.clean) - - def test_clean_dtf_several_supertypes_one_has_dtf_other_is_builtin_bad(self): - """ - Testing clean() on a DATETIMEFORMATted Datatype with two supertypes: STR and another DTFd Datatype. - """ - dtf = BasicConstraint.DATETIMEFORMAT - _super_DT, constr_DT = self._setup_inheriting_datatype( - "DateTimeDT", "String with a DATETIMEFORMAT", dtf, "%Y %b %d", self.STR, - "OverwritingDateTimeDT", "String with a DATETIMEFORMAT whose parent also has one", dtf, "%Y %d") - constr_DT.restricts.add(self.STR) - self.assertRaisesRegexp(ValidationError, - re.escape(('Datatype "{}" should have only one DATETIMEFORMAT restriction acting on ' - 'it, but it has {}'.format(constr_DT, 2))), - constr_DT.clean) - - def test_clean_float_conflicting_min_max_val_bad(self): - """ - Testing clean() on a float Datatype with conflicting MIN|MAX_VAL defined directly. - """ - constr_DT = self._setup_datatype( - "ConflictingBoundsDT", - "Float with conflicting MIN|MAX_VAL", - [(BasicConstraint.MIN_VAL, "15"), (BasicConstraint.MAX_VAL, "5")], - [self.FLOAT]) - - self.assertRaisesRegexp(ValidationError, - re.escape(('Datatype "{}" has effective MIN_VAL {} exceeding its effective MAX_VAL {}' - .format(constr_DT, 15, 5))), - constr_DT.clean) - - def test_clean_int_conflicting_inherited_min_max_val_bad(self): - """ - Testing clean() on an int Datatype with conflicting MIN|MAX_VAL defined on its supertypes. - """ - _, _, constr_DT = self._setup_inheriting_datatype2( - "BoundedFloatDT", "Float with a MIN_VAL", BasicConstraint.MIN_VAL, "20", self.FLOAT, - "BoundedIntDT", "Int with a MAX_VAL", BasicConstraint.MAX_VAL, "18.2", self.INT, - "InheritingBadBoundsDT", "Datatype inheriting conflicting MIN|MAX_VAL", None, None) - - self.assertRaisesRegexp(ValidationError, - re.escape(('Datatype "{}" has effective MIN_VAL {} exceeding its effective MAX_VAL {}' - .format(constr_DT, 20, 18.2))), - constr_DT.clean) - - def test_clean_float_conflicting_half_inherited_min_max_val_bad(self): - """ - Testing clean() on a float Datatype with conflicting MIN|MAX_VAL, one inherited and one directly. - """ - super_DT = Datatype(name="BoundedDT", description="Float with a MIN_VAL", user=self.myUser) - super_DT.full_clean() - super_DT.save() - super_DT.restricts.add(self.FLOAT) - super_DT.basic_constraints.create(ruletype=BasicConstraint.MIN_VAL, rule="17.7") - - constr_DT = Datatype(name="ConflictingBoundsDT", - description="Float with half-inherited conflicting MIN|MAX_VAL", - user=self.myUser) - constr_DT.full_clean() - constr_DT.save() - constr_DT.restricts.add(super_DT) - constr_DT.basic_constraints.create(ruletype=BasicConstraint.MAX_VAL, rule="6") - - self.assertRaisesRegexp(ValidationError, - re.escape(('Datatype "{}" has effective MIN_VAL {} exceeding its effective MAX_VAL {}' - .format(constr_DT, 17.7, 6))), - constr_DT.clean) - - def test_clean_int_min_max_val_too_narrow_bad(self): - """ - Testing clean() on an integer Datatype whose MIN|MAX_VAL do not admit any integers. - """ - constr_DT = Datatype(name="ConflictingBoundsDT", - description="INT with MIN|MAX_VAL too narrow", - user=self.myUser) - constr_DT.full_clean() - constr_DT.save() - constr_DT.restricts.add(self.INT) - constr_DT.basic_constraints.create(ruletype=BasicConstraint.MIN_VAL, rule="15.7") - constr_DT.basic_constraints.create(ruletype=BasicConstraint.MAX_VAL, rule="15.9") - - self.assertRaisesRegexp( - ValidationError, - re.escape((('Datatype "{}" has built-in type INT, but there are no integers between its ' - 'effective MIN_VAL {} and its effective MAX_VAL {}').format( - constr_DT, - 15.7, - 15.9))), - constr_DT.clean) - - def test_clean_int_inherited_min_max_val_too_narrow_bad(self): - """ - Testing clean() on an integer Datatype whose inherited MIN|MAX_VAL do not admit any integers. - """ - super_DT = Datatype(name="BoundedFloatDT", description="Float with a MIN_VAL", - user=self.myUser) - super_DT.full_clean() - super_DT.save() - super_DT.restricts.add(self.FLOAT) - super_DT.basic_constraints.create(ruletype=BasicConstraint.MIN_VAL, rule="20.2") - - second_DT = Datatype(name="BoundedIntDT", description="Int with a MAX_VAL", - user=self.myUser) - second_DT.full_clean() - second_DT.save() - second_DT.restricts.add(self.INT) - second_DT.basic_constraints.create(ruletype=BasicConstraint.MAX_VAL, rule="20.55") - - constr_DT = Datatype(name="InheritingBadBoundsDT", - description="Datatype inheriting too-narrow MIN|MAX_VAL", - user=self.myUser) - constr_DT.full_clean() - constr_DT.save() - constr_DT.restricts.add(super_DT) - constr_DT.restricts.add(second_DT) - - self.assertRaisesRegexp( - ValidationError, - re.escape((('Datatype "{}" has built-in type INT, but there ' - 'are no integers between its effective MIN_VAL {} ' - 'and its effective MAX_VAL {}').format( - constr_DT, - 20.2, - 20.55))), - constr_DT.clean) - - def test_clean_int_half_inherited_min_max_val_too_narrow_bad(self): - """ - Testing clean() on a float Datatype with half-inherited MIN|MAX_VAL that are too narrow. - """ - super_DT = Datatype(name="BoundedDT", description="Float with a MIN_VAL", user=self.myUser) - super_DT.full_clean() - super_DT.save() - super_DT.restricts.add(self.FLOAT) - super_DT.basic_constraints.create(ruletype=BasicConstraint.MIN_VAL, rule="17.1") - - constr_DT = Datatype(name="NarrowBoundsDT", - description="INT with half-inherited too-narrow MIN|MAX_VAL", - user=self.myUser) - constr_DT.full_clean() - constr_DT.save() - constr_DT.restricts.add(super_DT) - constr_DT.restricts.add(self.INT) - constr_DT.basic_constraints.create(ruletype=BasicConstraint.MAX_VAL, rule="17.8") - - self.assertRaisesRegexp( - ValidationError, - re.escape((('Datatype "{}" has built-in type INT, but there are no ' - 'integers between its effective MIN_VAL {} and its ' - 'effective MAX_VAL {}').format(constr_DT, 17.1, 17.8))), - constr_DT.clean) - - def test_clean_str_conflicting_min_max_length_bad(self): - """ - Testing clean() on a string Datatype with conflicting MIN|MAX_LENGTH defined directly. - """ - constr_DT = Datatype(name="ConflictingBoundsDT", - description="String with conflicting MIN|MAX_LENGTH", - user=self.myUser) - constr_DT.full_clean() - constr_DT.save() - constr_DT.restricts.add(self.STR) - constr_DT.basic_constraints.create(ruletype=BasicConstraint.MIN_LENGTH, rule="2234") - constr_DT.basic_constraints.create(ruletype=BasicConstraint.MAX_LENGTH, rule="6") - - self.assertRaisesRegexp( - ValidationError, - re.escape(('Datatype "{}" has effective MIN_LENGTH {} exceeding ' - 'its effective MAX_LENGTH {}'.format(constr_DT, 2234, 6))), - constr_DT.clean) - - def test_clean_str_conflicting_inherited_min_max_length_bad(self): - """ - Testing clean() on a string Datatype with conflicting MIN|MAX_LENGTH defined on its supertypes. - """ - super_DT = Datatype(name="BoundedMinDT", description="String with a MIN_LENGTH", user=self.myUser) - super_DT.full_clean() - super_DT.save() - super_DT.restricts.add(self.STR) - super_DT.basic_constraints.create(ruletype=BasicConstraint.MIN_LENGTH, rule="44") - - second_DT = Datatype(name="BoundedMaxDT", description="String with a MAX_LENGTH", user=self.myUser) - second_DT.full_clean() - second_DT.save() - second_DT.restricts.add(self.STR) - second_DT.basic_constraints.create(ruletype=BasicConstraint.MAX_LENGTH, rule="22") - - constr_DT = Datatype(name="InheritingBadBoundsDT", - description="Datatype inheriting conflicting MIN|MAX_LENGTH", - user=self.myUser) - constr_DT.full_clean() - constr_DT.save() - constr_DT.restricts.add(super_DT) - constr_DT.restricts.add(second_DT) - - self.assertRaisesRegexp( - ValidationError, - re.escape(('Datatype "{}" has effective MIN_LENGTH {} exceeding ' - 'its effective MAX_LENGTH {}'.format(constr_DT, 44, 22))), - constr_DT.clean) - - def test_clean_str_conflicting_half_inherited_min_max_length_bad(self): - """ - Testing clean() on a string Datatype with conflicting MIN|MAX_LENGTH, one inherited and one direct. - """ - super_DT = Datatype(name="BoundedDT", description="String with a MIN_LENGTH", - user=self.myUser) - super_DT.full_clean() - super_DT.save() - super_DT.restricts.add(self.STR) - super_DT.basic_constraints.create(ruletype=BasicConstraint.MAX_LENGTH, rule="20") - - constr_DT = Datatype(name="HalfInheritingBadBoundsDT", - description="Datatype inheriting conflicting MIN|MAX_LENGTH", - user=self.myUser) - constr_DT.full_clean() - constr_DT.save() - constr_DT.restricts.add(super_DT) - constr_DT.basic_constraints.create(ruletype=BasicConstraint.MIN_LENGTH, rule="30") - - self.assertRaisesRegexp( - ValidationError, - re.escape(('Datatype "{}" has effective MIN_LENGTH {} exceeding ' - 'its effective MAX_LENGTH {}'.format(constr_DT, 30, 20))), - constr_DT.clean) - - # FIXME: add some tests here when CustomConstraints are fully-coded. - - #### - # Tests of is_complete() and complete_clean(). - def test_is_complete_unsaved(self): - """ - Tests is_complete() on an unsaved Datatype (returns False). - """ - my_DT = Datatype(name="IncompleteDT", description="Non-finished Datatype", - user=self.myUser) - my_DT.full_clean() - - self.assertEquals(my_DT.is_complete(), False) - - def test_is_complete_incomplete(self): - """ - Tests is_complete() on a saved but incomplete Datatype (returns False). - """ - my_DT = Datatype(name="IncompleteDT", description="Non-finished Datatype", - user=self.myUser) - my_DT.full_clean() - my_DT.save() - - self.assertEquals(my_DT.is_complete(), False) - - def test_is_complete_restricts_string(self): - """ - Tests is_complete() on a complete Datatype that restricts STR (returns True). - """ - my_DT = Datatype(name="IncompleteDT", description="Non-finished Datatype", - user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(self.STR) - - self.assertEquals(my_DT.is_complete(), True) - - def test_is_complete_restricts_others(self): - """ - Tests is_complete() on a complete Datatype that restricts other Datatypes (returns True). - """ - super_DT = Datatype(name="SuperDT", description="Supertype", user=self.myUser) - super_DT.full_clean() - super_DT.save() - super_DT.restricts.add(self.STR) - - middle_DT = Datatype(name="MiddleDT", description="Middle type", user=self.myUser) - middle_DT.full_clean() - middle_DT.save() - middle_DT.restricts.add(super_DT) - - my_DT = Datatype(name="SubDT", description="Subtype", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(middle_DT, self.INT) - - self.assertEquals(middle_DT.is_complete(), True) - self.assertEquals(my_DT.is_complete(), True) - - self.assertEquals(my_DT.is_complete(), True) - - def test_complete_clean_unsaved_bad(self): - """ - Tests complete_clean() on an unsaved Datatype. - """ - my_DT = Datatype(name="IncompleteDT", description="Non-finished Datatype", user=self.myUser) - my_DT.full_clean() - - self.assertRaisesRegexp(ValidationError, - re.escape(('Datatype "{}" does not restrict any of the Shipyard atomic Datatypes' - .format(my_DT))), - my_DT.complete_clean) - - def test_complete_clean_incomplete(self): - """ - Tests complete_clean() on a saved but incomplete Datatype. - """ - my_DT = Datatype(name="IncompleteDT", description="Non-finished Datatype", user=self.myUser) - my_DT.full_clean() - my_DT.save() - - self.assertRaisesRegexp(ValidationError, - re.escape(('Datatype "{}" does not restrict any of the Shipyard atomic Datatypes' - .format(my_DT))), - my_DT.complete_clean) - - def test_complete_clean_restricts_string(self): - """ - Tests complete_clean() on a complete Datatype that restricts STR. - """ - my_DT = Datatype(name="IncompleteDT", description="Non-finished Datatype", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(self.STR) - - self.assertEquals(my_DT.complete_clean(), None) - - def test_complete_clean_restricts_others(self): - """ - Tests complete_clean() on a complete Datatype that restricts other Datatypes (returns True). - """ - super_DT = Datatype(name="SuperDT", description="Supertype", user=self.myUser) - super_DT.full_clean() - super_DT.save() - super_DT.restricts.add(self.STR) - - middle_DT = Datatype(name="MiddleDT", description="Middle type", user=self.myUser) - middle_DT.full_clean() - middle_DT.save() - middle_DT.restricts.add(super_DT) - - my_DT = Datatype(name="SubDT", description="Subtype", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(middle_DT, self.STR) - - self.assertEquals(middle_DT.complete_clean(), None) - self.assertEquals(my_DT.complete_clean(), None) - - self.assertEquals(my_DT.complete_clean(), None) - - # Quick check of propagation. - def test_complete_clean_propagate_from_clean(self): - """ - Testing complete_clean() on a string Datatype with conflicting MIN|MAX_LENGTH defined on its supertypes. - """ - super_DT = Datatype(name="BoundedMinDT", description="String with a MIN_LENGTH", user=self.myUser) - super_DT.full_clean() - super_DT.save() - super_DT.restricts.add(self.STR) - super_DT.basic_constraints.create(ruletype=BasicConstraint.MIN_LENGTH, rule="44") - - second_DT = Datatype(name="BoundedMaxDT", description="String with a MAX_LENGTH", user=self.myUser) - second_DT.full_clean() - second_DT.save() - second_DT.restricts.add(self.STR) - second_DT.basic_constraints.create(ruletype=BasicConstraint.MAX_LENGTH, rule="22") - - constr_DT = Datatype(name="InheritingBadBoundsDT", - description="Datatype inheriting conflicting MIN|MAX_LENGTH", - user=self.myUser) - constr_DT.full_clean() - constr_DT.save() - constr_DT.restricts.add(super_DT) - constr_DT.restricts.add(second_DT) - - self.assertRaisesRegexp(ValidationError, - re.escape(('Datatype "{}" has effective MIN_LENGTH {} exceeding its effective ' - 'MAX_LENGTH {}').format(constr_DT, 44, 22)), - constr_DT.complete_clean) - - -class DatatypeGetBuiltinTypeTests(MetadataTestCase): - """ - Tests of the Datatype.get_builtin_type() function. - """ - def test_on_builtins(self): - """ - Testing on the built-in Shipyard types. - """ - self.assertEquals(self.STR.get_builtin_type(), self.STR) - self.assertEquals(self.INT.get_builtin_type(), self.INT) - self.assertEquals(self.FLOAT.get_builtin_type(), self.FLOAT) - self.assertEquals(self.BOOL.get_builtin_type(), self.BOOL) - - ######## - def __test_on_direct_builtin_descendant_h(self, builtin_type): - """ - Helper for testing on direct descendants on the builtins. - """ - my_DT = Datatype(name="DescendantDT", description="Descendant of builtin DT", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(builtin_type) - - self.assertEquals(my_DT.get_builtin_type(), builtin_type) - - def test_direct_descendant_int(self): - """ - Testing case where Datatype is a descendant of INT. - """ - self.__test_on_direct_builtin_descendant_h(self.INT) - - def test_direct_descendant_str(self): - """ - Testing case where Datatype is a descendant of STR. - """ - self.__test_on_direct_builtin_descendant_h(self.STR) - - def test_direct_descendant_float(self): - """ - Testing case where Datatype is a descendant of FLOAT. - """ - self.__test_on_direct_builtin_descendant_h(self.FLOAT) - - def test_direct_descendant_bool(self): - """ - Testing case where Datatype is a descendant of BOOL. - """ - self.__test_on_direct_builtin_descendant_h(self.BOOL) - - ######## - def __test_supertype_precedence_h(self, builtin_types_to_restrict, most_restrictive_type): - """ - Helper for testing appropriate supertype precedence. - """ - my_DT = Datatype(name="InheritingDT", description="Datatype with several supertypes", user=self.myUser) - my_DT.full_clean() - my_DT.save() - - for to_restrict in builtin_types_to_restrict: - my_DT.restricts.add(to_restrict) - - self.assertEquals(my_DT.get_builtin_type(), most_restrictive_type) - - def test_supertype_precedence_float_over_str(self): - """ - FLOAT should take precedence over STR. - """ - self.__test_supertype_precedence_h([self.STR, self.FLOAT], self.FLOAT) - - def test_supertype_precedence_int_over_str(self): - """ - INT should take precedence over STR. - """ - self.__test_supertype_precedence_h([self.STR, self.INT], self.INT) - - def test_supertype_precedence_bool_over_str(self): - """ - BOOL should take precedence over STR. - """ - self.__test_supertype_precedence_h([self.BOOL, self.STR], self.BOOL) - - def test_supertype_precedence_int_over_float(self): - """ - INT should take precedence over FLOAT. - """ - self.__test_supertype_precedence_h([self.INT, self.FLOAT], self.INT) - - def test_supertype_precedence_bool_over_float(self): - """ - BOOL should take precedence over FLOAT. - """ - self.__test_supertype_precedence_h([self.FLOAT, self.BOOL], self.BOOL) - - def test_supertype_precedence_bool_over_int(self): - """ - BOOL should take precedence over INT. - """ - self.__test_supertype_precedence_h([self.INT, self.BOOL], self.BOOL) - - def test_supertype_precedence_multiple(self): - """ - Testing precendence when there are several builtins restricted. - """ - self.__test_supertype_precedence_h([self.INT, self.BOOL, self.STR], self.BOOL) - - ######## - def test_multiple_supertypes(self): - """ - Testing case where Datatype has multiple supertypes of varying generations. - """ - super_DT = Datatype(name="SuperDT", description="Super DT", user=self.myUser) - super_DT.full_clean() - super_DT.save() - super_DT.restricts.add(self.FLOAT) - - super2_DT = Datatype(name="SuperDT2", description="Super DT 2", user=self.myUser) - super2_DT.full_clean() - super2_DT.save() - super2_DT.restricts.add(self.STR) - - super3_DT = Datatype(name="SuperDT3", description="Super DT 3", user=self.myUser) - super3_DT.full_clean() - super3_DT.save() - super3_DT.restricts.add(super_DT) - - my_DT = Datatype(name="DescendantDT", description="Descendant of several supertypes", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(super2_DT) - my_DT.restricts.add(super3_DT) - - self.assertEquals(my_DT.get_builtin_type(), self.FLOAT) - - def test_multiple_supertypes_2(self): - """ - Another testing case where Datatype has multiple supertypes of varying generations. - """ - super_DT = Datatype(name="SuperDT", description="Super DT", user=self.myUser) - super_DT.full_clean() - super_DT.save() - super_DT.restricts.add(self.FLOAT) - - super2_DT = Datatype(name="SuperDT2", description="Super DT 2", user=self.myUser) - super2_DT.full_clean() - super2_DT.save() - super2_DT.restricts.add(self.BOOL) - - super3_DT = Datatype(name="SuperDT3", description="Super DT 3", user=self.myUser) - super3_DT.full_clean() - super3_DT.save() - super3_DT.restricts.add(super_DT) - - my_DT = Datatype(name="DescendantDT", description="Descendant of several supertypes", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(super2_DT) - my_DT.restricts.add(super3_DT) - - self.assertEquals(my_DT.get_builtin_type(), self.BOOL) - - -class DatatypeCheckBasicConstraints(MetadataTestCase): - """ - Tests of Datatype.check_basic_constraints(). - """ - def __test_builtin_type_good_h(self, builtin_type, string_to_check): - """ - Helper for testing good cases where the input conforms to the appropriate built-in type. - """ - my_DT = Datatype(name="MyDT", description="Non-builtin datatype", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(builtin_type) - - # Check builtin type too. - self.assertEquals(builtin_type.check_basic_constraints(string_to_check), []) - self.assertEquals(my_DT.check_basic_constraints(string_to_check), []) - - my_DT.delete() - - def test_str_good(self): - """ - Testing case of a string with no constraints. - """ - self.__test_builtin_type_good_h(self.STR, "foo") - - def test_float_good(self): - """ - Testing case of a float with no constraints. - """ - self.__test_builtin_type_good_h(self.FLOAT, "3.14") - - def test_int_good(self): - """ - Testing case of an int with no constraints. - """ - self.__test_builtin_type_good_h(self.INT, "-8") - - def test_bool_good(self): - """ - Testing case of an int with no constraints. - """ - self.__test_builtin_type_good_h(self.BOOL, "True") - self.__test_builtin_type_good_h(self.BOOL, "TRUE") - self.__test_builtin_type_good_h(self.BOOL, "true") - self.__test_builtin_type_good_h(self.BOOL, "T") - self.__test_builtin_type_good_h(self.BOOL, "t") - self.__test_builtin_type_good_h(self.BOOL, "1") - self.__test_builtin_type_good_h(self.BOOL, "False") - self.__test_builtin_type_good_h(self.BOOL, "FALSE") - self.__test_builtin_type_good_h(self.BOOL, "false") - self.__test_builtin_type_good_h(self.BOOL, "F") - self.__test_builtin_type_good_h(self.BOOL, "f") - self.__test_builtin_type_good_h(self.BOOL, "0") - - def __test_builtin_type_bad_h(self, builtin_type, string_to_check): - """ - Helper for testing cases where the input does not conform to the appropriate built-in type. - """ - my_DT = Datatype(name="MyDT", description="Non-builtin datatype", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(builtin_type) - - builtin_type_error = "" - if builtin_type == self.FLOAT: - builtin_type_error = "Was not float" - elif builtin_type == self.INT: - builtin_type_error = "Was not integer" - elif builtin_type == self.BOOL: - builtin_type_error = "Was not Boolean" - - # Check builtin type too. - self.assertEquals(builtin_type.check_basic_constraints(string_to_check), [builtin_type_error]) - self.assertEquals(my_DT.check_basic_constraints(string_to_check), [builtin_type_error]) - - def test_float_error(self): - """ - Testing case where string cannot be cast to a float. - """ - self.__test_builtin_type_bad_h(self.FLOAT, "foo") - - def test_int_error(self): - """ - Testing case where string cannot be cast to an int. - """ - self.__test_builtin_type_bad_h(self.INT, "1.72") - - def test_bool_error(self): - """ - Testing case where string cannot be cast to a Boolean. - """ - self.__test_builtin_type_bad_h(self.BOOL, "maybe") - - # Test that "Was not [builtin type]" overrules other constraints. - def __test_builtin_type_with_constraint_bad_h(self, builtin_type, BC_type, constr_val, string_to_check): - """ - Helper for testing cases where the input does not conform to the appropriate built-in type. - """ - my_DT = Datatype(name="MyDT", description="Non-builtin datatype", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(builtin_type) - my_DT.basic_constraints.create(ruletype=BC_type, rule="{}".format(constr_val)) - - builtin_type_error = "" - if builtin_type == self.FLOAT: - builtin_type_error = "Was not float" - elif builtin_type == self.INT: - builtin_type_error = "Was not integer" - elif builtin_type == self.BOOL: - builtin_type_error = "Was not Boolean" - - self.assertEquals(my_DT.check_basic_constraints(string_to_check), [builtin_type_error]) - - def test_float_error_with_constraint(self): - """ - Testing case where string cannot be cast to a float and the Datatype has a constraint. - """ - self.__test_builtin_type_with_constraint_bad_h(self.FLOAT, BasicConstraint.MIN_VAL, 8, "foo") - - def test_int_error_with_constraint(self): - """ - Testing case where string cannot be cast to an integer and the Datatype has a constraint. - """ - self.__test_builtin_type_with_constraint_bad_h(self.INT, BasicConstraint.MAX_VAL, 17, "1.2") - - def test_bool_error_with_constraint(self): - """ - Testing case where string cannot be cast to an integer and the Datatype has a constraint. - """ - self.__test_builtin_type_with_constraint_bad_h(self.BOOL, BasicConstraint.REGEXP, ".*", "what") - - ######## - def __test_numerical_constraint_h(self, builtin_type, BC_type, constr_val, string_to_check, - passes_constraint=True): - """ - Helper to test strings against numerical constraints. - """ - my_DT = Datatype(name="MyDT", description="Datatype with numerical BC", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(builtin_type) - - my_BC = my_DT.basic_constraints.create(ruletype=BC_type, rule="{}".format(constr_val)) - - if passes_constraint: - self.assertEquals(my_DT.check_basic_constraints(string_to_check), []) - else: - self.assertEquals(my_DT.check_basic_constraints(string_to_check), [my_BC]) - - #### - def test_min_length_pass(self): - """ - Testing case where a string passes the MIN_LENGTH restriction. - """ - self.__test_numerical_constraint_h(self.STR, BasicConstraint.MIN_LENGTH, 5, "foobar", - passes_constraint=True) - - def test_min_length_edge_pass(self): - """ - Testing case where a string just passes (edge-condition) the MIN_LENGTH restriction. - """ - self.__test_numerical_constraint_h(self.STR, BasicConstraint.MIN_LENGTH, 11, "hello world", - passes_constraint=True) - - def test_min_length_fail(self): - """ - Testing case where a string fails the MIN_LENGTH restriction. - """ - self.__test_numerical_constraint_h(self.STR, BasicConstraint.MIN_LENGTH, 100, "short string", - passes_constraint=False) - - def test_min_length_edge_fail(self): - """ - Testing case where a string just fails (edge-condition) the MIN_LENGTH restriction. - """ - self.__test_numerical_constraint_h(self.STR, BasicConstraint.MIN_LENGTH, 8, "bye all", - passes_constraint=False) - - #### - def test_max_length_pass(self): - """ - Testing case where a string passes the MAX_LENGTH restriction. - """ - self.__test_numerical_constraint_h(self.STR, BasicConstraint.MAX_LENGTH, 2, "Hi", - passes_constraint=True) - - def test_max_length_edge_pass(self): - """ - Testing case where a string just passes (edge-condition) the MAX_LENGTH restriction. - """ - self.__test_numerical_constraint_h(self.STR, BasicConstraint.MAX_LENGTH, 27, "onetwothreefourfive and six", - passes_constraint=True) - - def test_max_length_fail(self): - """ - Testing case where a string fails the MAX_LENGTH restriction. - """ - self.__test_numerical_constraint_h(self.STR, BasicConstraint.MAX_LENGTH, 10, "Hello everyone", - passes_constraint=False) - - def test_max_length_edge_fail(self): - """ - Testing case where a string just fails (edge-condition) the MAX_LENGTH restriction. - """ - self.__test_numerical_constraint_h(self.STR, BasicConstraint.MAX_LENGTH, 10, "Hello world", - passes_constraint=False) - - #### - def test_min_val_float_pass(self): - """ - Testing case where a float passes the MIN_VAL restriction. - """ - self.__test_numerical_constraint_h(self.FLOAT, BasicConstraint.MIN_VAL, 17, "100", - passes_constraint=True) - - def test_min_val_float_edge_pass(self): - """ - Testing case where a float just passes (edge-condition) the MIN_VAL restriction. - """ - self.__test_numerical_constraint_h(self.FLOAT, BasicConstraint.MIN_VAL, -1722.4, "-1722.4", - passes_constraint=True) - - def test_min_val_float_fail(self): - """ - Testing case where a float fails the MIN_VAL restriction. - """ - self.__test_numerical_constraint_h(self.FLOAT, BasicConstraint.MIN_VAL, 17, "14", - passes_constraint=False) - - # Note that there isn't an "edge fail" case here. - - #### - def test_max_val_float_pass(self): - """ - Testing case where a float passes the MAX_VAL restriction. - """ - self.__test_numerical_constraint_h(self.FLOAT, BasicConstraint.MAX_VAL, -100090, "-111117.445", - passes_constraint=True) - - def test_max_val_float_edge_pass(self): - """ - Testing case where a float just passes (edge-condition) the MIN_VAL restriction. - """ - self.__test_numerical_constraint_h(self.FLOAT, BasicConstraint.MAX_VAL, 42.77, "42.77", - passes_constraint=True) - - def test_max_val_float_fail(self): - """ - Testing case where a float fails the MAX_VAL restriction. - """ - self.__test_numerical_constraint_h(self.FLOAT, BasicConstraint.MAX_VAL, -17, "-1", - passes_constraint=False) - # As above there is no "edge fail" here. - - #### - def test_min_val_int_pass(self): - """ - Testing case where an integer passes the MIN_VAL restriction. - """ - self.__test_numerical_constraint_h(self.INT, BasicConstraint.MIN_VAL, -4, "6", - passes_constraint=True) - - def test_min_val_int_edge_pass(self): - """ - Testing case where an integer just passes (edge-condition) the MIN_VAL restriction. - """ - self.__test_numerical_constraint_h(self.INT, BasicConstraint.MIN_VAL, 165, "165", - passes_constraint=True) - - def test_min_val_int_fail(self): - """ - Testing case where an integer fails the MIN_VAL restriction. - """ - self.__test_numerical_constraint_h(self.FLOAT, BasicConstraint.MIN_VAL, 3, "-2", - passes_constraint=False) - - def test_min_val_int_edge_fail(self): - """ - Testing case where an integer just fails (edge-condition) the MIN_VAL restriction. - """ - self.__test_numerical_constraint_h(self.FLOAT, BasicConstraint.MIN_VAL, 7, "6", - passes_constraint=False) - - #### - def test_max_val_int_pass(self): - """ - Testing case where an integer passes the MAX_VAL restriction. - """ - self.__test_numerical_constraint_h(self.INT, BasicConstraint.MAX_VAL, 85, "3", - passes_constraint=True) - - def test_max_val_int_edge_pass(self): - """ - Testing case where an integer just passes (edge-condition) the MAX_VAL restriction. - """ - self.__test_numerical_constraint_h(self.INT, BasicConstraint.MAX_VAL, -92, "-92", - passes_constraint=True) - - def test_max_val_int_fail(self): - """ - Testing case where an integer fails the MAX_VAL restriction. - """ - self.__test_numerical_constraint_h(self.FLOAT, BasicConstraint.MAX_VAL, 3, "44", - passes_constraint=False) - - def test_max_val_int_edge_fail(self): - """ - Testing case where an integer just fails (edge-condition) the MAX_VAL restriction. - """ - self.__test_numerical_constraint_h(self.FLOAT, BasicConstraint.MAX_VAL, 7, "8", - passes_constraint=False) - - #### - def __test_regexp_h(self, builtin_type, constr_val, string_to_check, - passes_constraint=True): - """ - Helper to test strings against a REGEXP constraints. - """ - my_DT = Datatype(name="MyDT", description="Datatype with REGEXP BC", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(builtin_type) - - my_BC = my_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule="{}".format(constr_val)) - - if passes_constraint: - self.assertEquals(my_DT.check_basic_constraints(string_to_check), []) - else: - self.assertEquals(my_DT.check_basic_constraints(string_to_check), [my_BC]) - - def test_str_regexp_pass(self): - """ - Test a string against a REGEXP it satisfies. - """ - self.__test_regexp_h(self.STR, "[a-z]+", "123abc", passes_constraint=True) - - def test_str_regexp_fail(self): - """ - Test a string against a REGEXP it does not satisfy. - """ - self.__test_regexp_h(self.STR, "foo|bar", "123abc", passes_constraint=False) - - def test_float_regexp_pass(self): - """ - Test a float against a REGEXP it satisfies. - """ - self.__test_regexp_h(self.FLOAT, "[1-9]+\.663", "1325.663", passes_constraint=True) - - def test_float_regexp_fail(self): - """ - Test a float against a REGEXP it doesn't satisfy. - """ - self.__test_regexp_h(self.FLOAT, "1065[0-9]+", "132544", passes_constraint=False) - - def test_int_regexp_pass(self): - """ - Test an int against a REGEXP it satisfies. - """ - self.__test_regexp_h(self.INT, ".+", "4444", passes_constraint=True) - - def test_int_regexp_fail(self): - """ - Test an int against a REGEXP it doesn't satisfy. - """ - self.__test_regexp_h(self.INT, "[1-9]{4}", "-1000", passes_constraint=False) - - def test_bool_regexp_pass(self): - """ - Test a Boolean against a REGEXP it satisfies. - """ - self.__test_regexp_h(self.BOOL, "True|TRUE|true|t|1", "True", passes_constraint=True) - - def test_bool_regexp_fail(self): - """ - Test a Boolean against a REGEXP it doesn't satisfy. - """ - self.__test_regexp_h(self.STR, "False", "True", passes_constraint=False) - - #### - # Some test cases with combined restrictions. - def test_str_multiple_restrictions_pass(self): - """ - Test a string against several restrictions. - """ - my_DT = Datatype(name="MyDT", description="Datatype with several restrictions", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(self.STR) - - my_DT.basic_constraints.create(ruletype=BasicConstraint.MIN_LENGTH, rule="4") - my_DT.basic_constraints.create(ruletype=BasicConstraint.MAX_LENGTH, rule="7") - my_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule="foo...") - my_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule="...bar") - - self.assertEquals(my_DT.check_basic_constraints("foobar"), []) - - def test_str_multiple_restrictions_fail(self): - """ - Test a string against several restrictions, some of which fail. - """ - my_DT = Datatype(name="MyDT", description="Datatype with several restrictions", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(self.STR) - - my_DT.basic_constraints.create(ruletype=BasicConstraint.MIN_LENGTH, rule="4") - my_max_length = my_DT.basic_constraints.create(ruletype=BasicConstraint.MAX_LENGTH, rule="5") - my_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule="foo...") - my_regexp_2 = my_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule="...baz") - - constr_fail = my_DT.check_basic_constraints("foobar") - self.assertEquals(len(constr_fail), 2) - self.assertEquals(my_max_length in constr_fail, True) - self.assertEquals(my_regexp_2 in constr_fail, True) - - def test_float_multiple_restrictions_pass(self): - """ - Test a float against several restrictions, all of which pass. - """ - my_DT = Datatype(name="MyDT", description="Datatype with several restrictions", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(self.FLOAT) - - my_DT.basic_constraints.create(ruletype=BasicConstraint.MIN_VAL, rule="1999") - my_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule="^....$") - my_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule="..14") - - self.assertEquals(my_DT.check_basic_constraints("2014"), []) - - def test_float_multiple_restrictions_fail(self): - """ - Test a float against several restrictions, some of which fail. - """ - my_DT = Datatype(name="MyDT", description="Datatype with several restrictions", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(self.FLOAT) - - my_min_val = my_DT.basic_constraints.create(ruletype=BasicConstraint.MAX_VAL, rule="1999") - my_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule="^....$") - my_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule="..14") - - self.assertEquals(my_DT.check_basic_constraints("2014"), [my_min_val]) - - def test_int_multiple_restrictions_pass(self): - """ - Test an int against several restrictions, all of which pass. - """ - my_DT = Datatype(name="MyDT", description="Datatype with several restrictions", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(self.INT) - - my_DT.basic_constraints.create(ruletype=BasicConstraint.MAX_VAL, rule="2099") - my_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule="^....$") - my_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule="..35") - - self.assertEquals(my_DT.check_basic_constraints("2035"), []) - - def test_int_multiple_restrictions_fail(self): - """ - Test an int against several restrictions, some of which fail. - """ - my_DT = Datatype(name="MyDT", description="Datatype with several restrictions", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(self.INT) - - my_min_val = my_DT.basic_constraints.create(ruletype=BasicConstraint.MIN_VAL, rule="2099") - my_regexp = my_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule="^....$") - my_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule="35") - - constr_fail = my_DT.check_basic_constraints("935") - self.assertEquals(len(constr_fail), 2) - self.assertEquals(my_regexp in constr_fail, True) - self.assertEquals(my_min_val in constr_fail, True) - - def test_bool_multiple_restrictions_pass(self): - """ - Test a Boolean against several restrictions, all of which pass. - """ - my_DT = Datatype(name="MyDT", description="Datatype with several restrictions", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(self.BOOL) - - my_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule="T...") - my_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule="rue|RUE") - - self.assertEquals(my_DT.check_basic_constraints("True"), []) - - def test_bool_multiple_restrictions_fail(self): - """ - Test a Boolean against several restrictions, some of which fail. - """ - my_DT = Datatype(name="MyDT", description="Datatype with several restrictions", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(self.BOOL) - - my_regexp = my_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule="T...") - my_regexp_2 = my_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule="rue|RUE") - - constr_fail = my_DT.check_basic_constraints("False") - self.assertEquals(len(constr_fail), 2) - self.assertEquals(my_regexp in constr_fail, True) - self.assertEquals(my_regexp_2 in constr_fail, True) - - #### - # A couple of test cases for inherited constraints. - - # FIXME we need to think on this further! - - def test_str_inherit_restrictions(self): - """ - Testing a string against some inherited restrictions. - """ - super_DT = Datatype(name="SuperDT", description="Supertype", user=self.myUser) - super_DT.full_clean() - super_DT.save() - super_DT.restricts.add(self.STR) - my_regexp = super_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule="Hello t....") - - my_DT = Datatype(name="MyDT", description="Datatype inheriting a restriction", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(super_DT) - my_max_length = my_DT.basic_constraints.create(ruletype=BasicConstraint.MAX_LENGTH, rule="12") - - self.assertEquals(my_DT.check_basic_constraints("Hello there"), []) - self.assertEquals(my_DT.check_basic_constraints("Hello theremin"), [my_max_length]) - self.assertEquals(my_DT.check_basic_constraints("Hello"), [my_regexp]) - - constr_fail = my_DT.check_basic_constraints("Goodbye everyone") - self.assertEquals(len(constr_fail), 2) - self.assertEquals(my_regexp in constr_fail, True) - self.assertEquals(my_max_length in constr_fail, True) - - def test_float_inherit_restrictions(self): - """ - Testing a float against some inherited restrictions. - """ - super_DT = Datatype(name="SuperDT", description="Supertype", user=self.myUser) - super_DT.full_clean() - super_DT.save() - super_DT.restricts.add(self.STR) - super_DT.basic_constraints.create(ruletype=BasicConstraint.MIN_LENGTH, rule="2") - - my_DT = Datatype(name="MyDT", description="Datatype inheriting a restriction", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(super_DT) - my_DT.restricts.add(self.FLOAT) - my_max_val = my_DT.basic_constraints.create(ruletype=BasicConstraint.MAX_VAL, rule="95") - - self.assertEquals(my_DT.check_basic_constraints("82"), []) - self.assertEquals(my_DT.check_basic_constraints(".7"), []) - self.assertEquals(my_DT.check_basic_constraints("99"), [my_max_val]) - - # Note that since my_DT is no longer a STR, only my_max_val applies. - self.assertEquals(my_DT.check_basic_constraints("114"), [my_max_val]) - - def test_int_inherit_restrictions(self): - """ - Testing an integer against some inherited restrictions. - """ - super_DT = Datatype(name="SuperDT", description="Supertype", user=self.myUser) - super_DT.full_clean() - super_DT.save() - super_DT.restricts.add(self.STR) - my_regexp = super_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule="1000...") - - super2_DT = Datatype(name="SuperDT2", description="Supertype 2", user=self.myUser) - super2_DT.full_clean() - super2_DT.save() - super2_DT.restricts.add(self.INT) - my_min_val = super2_DT.basic_constraints.create(ruletype=BasicConstraint.MIN_VAL, rule="1000100") - - my_DT = Datatype(name="MyDT", description="Datatype inheriting restrictions", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(super_DT) - my_DT.restricts.add(super2_DT) - - self.assertEquals(my_DT.check_basic_constraints("1000107"), []) - self.assertEquals(my_DT.check_basic_constraints("1000004"), [my_min_val]) - self.assertEquals(my_DT.check_basic_constraints("1099999"), [my_regexp]) - - constr_fail = my_DT.check_basic_constraints("99999") - self.assertEquals(len(constr_fail), 2) - self.assertEquals(my_regexp in constr_fail, True) - self.assertEquals(my_min_val in constr_fail, True) - - def test_int_inherit_overridden_restriction(self): - """ - Testing an integer against an overridden inherited restriction. - """ - super_DT = Datatype(name="SuperDT", description="Supertype", user=self.myUser) - super_DT.full_clean() - super_DT.save() - super_DT.restricts.add(self.INT) - super_DT.basic_constraints.create(ruletype=BasicConstraint.MAX_VAL, rule="999") - - my_DT = Datatype(name="MyDT", description="Datatype inheriting restrictions", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(super_DT) - my_max_val = my_DT.basic_constraints.create(ruletype=BasicConstraint.MAX_VAL, rule="899") - - self.assertEquals(my_DT.check_basic_constraints("0"), []) - self.assertEquals(my_DT.check_basic_constraints("950"), [my_max_val]) - # super_max_val is overridden so only my_max_val should fail. - self.assertEquals(my_DT.check_basic_constraints("1055"), [my_max_val]) - - def test_bool_inherit_restrictions(self): - """ - Testing a Boolean against some inherited restrictions. - """ - super_DT = Datatype(name="SuperDT", description="Supertype", user=self.myUser) - super_DT.full_clean() - super_DT.save() - super_DT.restricts.add(self.BOOL) - my_regexp = super_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule="T.+") - my_regexp2 = super_DT.basic_constraints.create(ruletype=BasicConstraint.REGEXP, rule=".rue") - - my_DT = Datatype(name="MyDT", description="Datatype inheriting restrictions", user=self.myUser) - my_DT.full_clean() - my_DT.save() - my_DT.restricts.add(super_DT) - my_DT.restricts.add(self.BOOL) - - self.assertEquals(my_DT.check_basic_constraints("True"), []) - self.assertEquals(my_DT.check_basic_constraints("true"), [my_regexp]) - self.assertEquals(my_DT.check_basic_constraints("TRUE"), [my_regexp2]) - - self.assertEquals(set(my_DT.check_basic_constraints("False")), set([my_regexp, my_regexp2])) - - -class CompoundDatatypeMemberTests(MetadataTestCase): - def test_cdtMember_unicode(self): - """ - Unicode of compoundDatatypeMember should return - (column index, datatype name, column name) - """ - self.assertEqual( - unicode(self.test_cdt.members.get(column_idx=1)), - "label: string" - ) - self.assertEqual( - unicode(self.test_cdt.members.get(column_idx=2)), - "PBMCseq: DNANucSeq" - ) - self.assertEqual( - unicode(self.test_cdt.members.get(column_idx=3)), - "PLAseq: RNANucSeq" - ) - - -class CompoundDatatypeTests(MetadataTestCase): - - def test_cdt_zero_member_unicode(self): - """ - Unicode of empty CompoundDatatype should be empty. - """ - empty_cdt = CompoundDatatype(user=self.myUser) - empty_cdt.save() - self.assertEqual(unicode(empty_cdt), "[empty CompoundDatatype]") - - def test_cdt_single_member_unicode(self): - """ - Unicode on single-member cdt returns its member. - """ - self.assertEqual(unicode(self.DNAinput_cdt), - "(SeqToComplement: DNANucSeq)") - - def test_cdt_multiple_members_unicode(self): - """ - Unicode returns a list of its Datatype members. - - Each member is in the form of unicode(CompoundDatatypeMember). - """ - self.assertEqual( - unicode(self.test_cdt), - "(label: string, PBMCseq: DNANucSeq, PLAseq: RNANucSeq)") - - def test_cdt_four_members_short_name(self): - self.basic_cdt.members.get(column_idx=5).delete() - self.assertEqual( - self.basic_cdt.short_name, - "(label: string, integer: integer, float: float, bool: boolean)") - - def test_cdt_five_members_short_name(self): - self.assertEqual( - self.basic_cdt.short_name, - "(label: string, integer: integer, float: float, plus 2 others)") - - def test_clean_single_index_good(self): - """ - CompoundDatatype with single index equalling 1. - """ - sad_cdt = CompoundDatatype(user=self.myUser) - sad_cdt.save() - sad_cdt.members.create(datatype=self.RNA_dt, - column_name="ColumnTwo", - column_idx=1) - self.assertEqual(sad_cdt.clean(), None) - - def test_clean_single_index_bad(self): - """ - CompoundDatatype with single index not equalling 1. - """ - sad_cdt = CompoundDatatype(user=self.myUser) - sad_cdt.save() - sad_cdt.members.create(datatype=self.RNA_dt, - column_name="ColumnTwo", - column_idx=3) - - self.assertRaisesRegexp( - ValidationError, - re.escape(('Column indices of CompoundDatatype "{}" are not ' - 'consecutive starting from 1'.format(sad_cdt))), - sad_cdt.clean) - - def test_clean_consecutive_member_indices_correct(self): - """ - A CompoundDatatype with consecutive member indices passes clean. - """ - self.assertEqual(self.test_cdt.clean(), None) - - good_cdt = CompoundDatatype(user=self.myUser) - good_cdt.save() - good_cdt.members.create(datatype=self.RNA_dt, column_name="ColumnTwo", column_idx=2) - good_cdt.members.create(datatype=self.DNA_dt, column_name="ColumnOne", column_idx=1) - self.assertEqual(good_cdt.clean(), None) - - def test_clean_catches_consecutive_member_indices(self): - """ - A CompoundDatatype without consecutive member indices throws a ValidationError. - """ - bad_cdt = CompoundDatatype(user=self.myUser) - bad_cdt.save() - bad_cdt.members.create(datatype=self.RNA_dt, column_name="ColumnOne", column_idx=3) - bad_cdt.members.create(datatype=self.DNA_dt, column_name="ColumnTwo", column_idx=1) - - self.assertRaisesRegexp( - ValidationError, - re.escape(('Column indices of CompoundDatatype "{}" are not ' - 'consecutive starting from 1'.format(bad_cdt))), - bad_cdt.clean) - - def test_clean_members_no_column_names(self): - """ - Datatype members must have column names. - """ - cdt = CompoundDatatype(user=self.myUser) - cdt.save() - cdt.members.create(datatype=self.RNA_dt, column_idx=1) - self.assertRaisesRegexp(ValidationError, - "{'column_name': \[u'This field cannot be blank.'\]}", - cdt.clean) - - def test_copy_users_allowed(self): - cdt_without_permissions = CompoundDatatype.objects.filter( - users_allowed__isnull=True).first() - cdt_with_permissions = CompoundDatatype.objects.filter( - users_allowed__isnull=False).first() - self.assertIsNotNone(cdt_without_permissions) - self.assertIsNotNone(cdt_with_permissions) - expected_permissions = set( - cdt_with_permissions.users_allowed.values_list('username')) - - cdt_without_permissions.copy_permissions(cdt_with_permissions) - - permissions = set( - cdt_without_permissions.users_allowed.values_list('username')) - - self.assertEqual(expected_permissions, permissions) - - def test_copy_groups_allowed(self): - cdt_without_permissions = CompoundDatatype.objects.filter( - groups_allowed__isnull=True).first() - cdt_with_permissions = CompoundDatatype.objects.filter( - groups_allowed__isnull=False).first() - self.assertIsNotNone(cdt_without_permissions) - self.assertIsNotNone(cdt_with_permissions) - expected_permissions = set( - cdt_with_permissions.groups_allowed.values_list('name')) - - cdt_without_permissions.copy_permissions(cdt_with_permissions) - - permissions = set( - cdt_without_permissions.groups_allowed.values_list('name')) - - self.assertEqual(expected_permissions, permissions) - - def test_create_dataset_raw(self): - """ - Creating a raw Dataset should pass clean. - """ - path = os.path.join(samplecode_path, "doublet_cdt.csv") - raw_dataset = Dataset.create_dataset(file_path=path, user=self.myUser, keep_file=True, - name="something", description="desc") - self.assertEqual(raw_dataset.clean(), None) - - def test_create_dataset_valid(self): - """ - Creating a Dataset with a CDT, where the file conforms, should be OK. - """ - path = os.path.join(samplecode_path, "doublet_cdt.csv") - doublet_dataset = Dataset.create_dataset(file_path=path, user=self.myUser, cdt=self.doublet_cdt, - keep_file=True, name="something", description="desc") - self.assertEqual(doublet_dataset.clean(), None) - self.assertEqual(doublet_dataset.structure.clean(), None) - - def test_create_dataset_bad_num_cols(self): - """Define a dataset, but with the wrong number of headers.""" - path = os.path.join(samplecode_path, "step_0_triplet_3_rows.csv") - self.assertRaisesRegexp( - ValueError, - re.escape('The header of file "{}" does not match the CompoundDatatype "{}"' - .format(path, self.doublet_cdt)), - lambda: Dataset.create_dataset(file_path=path, user=self.myUser, cdt=self.doublet_cdt, - name="DS1", description="DS1 desc")) - - def test_create_dataset_bad_col_names(self): - """Define a dataset with the right number of header columns, but the wrong column names.""" - path = os.path.join(samplecode_path, "three_random_columns.csv") - self.assertRaisesRegexp( - ValueError, - re.escape('The header of file "{}" does not match the CompoundDatatype "{}"' - .format(path, self.triplet_cdt)), - lambda: Dataset.create_dataset(file_path=path, user=self.myUser, cdt=self.triplet_cdt, - name="DS1", description="DS1 desc")) - - def test_type_constraints_row(self): - - # The cdt schema is (string, int, float, bool, rna) - t1 = self.basic_cdt.check_constraints(['Once', 'upon', 'a', 'time', 'there']) - t2 = self.basic_cdt.check_constraints(['was', '1', 'young', 'lazy', 'dev']) - t3 = self.basic_cdt.check_constraints(['that', 'needed', '2', 'test', 'his']) - t4 = self.basic_cdt.check_constraints(['datatype', 'as', 'a', 'True', 'which']) - t5 = self.basic_cdt.check_constraints(['often', 'made', 'him', 'scream', 'UGGGG']) - - int_fail = u'Was not integer' - float_fail = u'Was not float' - bool_fail = u'Was not Boolean' - rna_fail = u"Failed check 'regexp=^[ACGUacgu]*$'" - - self.assertEqual(t1, [[], [int_fail], [float_fail], [bool_fail], [rna_fail]]) - self.assertEqual(t2, [[], [], [float_fail], [bool_fail], [rna_fail]]) - self.assertEqual(t3, [[], [int_fail], [], [bool_fail], [rna_fail]]) - self.assertEqual(t4, [[], [int_fail], [float_fail], [], [rna_fail]]) - self.assertEqual(t5, [[], [int_fail], [float_fail], [bool_fail], []]) - - -@skipIfDBFeature('is_mocked') -class DatatypeApiTests(TestCase): - - def setUp(self): - self.factory = APIRequestFactory() - self.kive_user = kive_user() - - self.list_path = reverse("datatype-list") - self.detail_pk = 7 - self.detail_path = reverse("datatype-detail", - kwargs={'pk': self.detail_pk}) - self.removal_path = reverse("datatype-removal-plan", - kwargs={'pk': self.detail_pk}) - - # This should equal metadata.ajax.CompoundDatatypeViewSet.as_view({"get": "list"}). - self.list_view, _, _ = resolve(self.list_path) - self.detail_view, _, _ = resolve(self.detail_path) - self.removal_view, _, _ = resolve(self.removal_path) - - def test_auth(self): - # First try to access while not logged in. - request = self.factory.get(self.list_path) - response = self.list_view(request) - self.assertEquals(response.data["detail"], - "Authentication credentials were not provided.") - - # Now log in and check that "detail" is not passed in the response. - force_authenticate(request, user=self.kive_user) - response = self.list_view(request) - self.assertNotIn('detail', response.data) - - def test_list(self): - """ - Test the CompoundDatatype API list view. - """ - request = self.factory.get(self.list_path) - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - # There are four CDTs loaded into the Database by default. - self.assertEquals(len(response.data), 7) - self.assertEquals(response.data[0]['id'], 1) - self.assertEquals(response.data[2]['name'], 'float') - - def test_detail(self): - request = self.factory.get(self.detail_path) - force_authenticate(request, user=self.kive_user) - response = self.detail_view(request, pk=self.detail_pk) - self.assertEquals(response.data['name'], 'nucleotide sequence') - - def test_removal_plan(self): - request = self.factory.get(self.removal_path) - force_authenticate(request, user=self.kive_user) - response = self.removal_view(request, pk=self.detail_pk) - self.assertEquals(response.data['Datatypes'], 1) - - def test_removal(self): - start_count = Datatype.objects.all().count() - - request = self.factory.delete(self.detail_path) - force_authenticate(request, user=self.kive_user) - response = self.detail_view(request, pk=self.detail_pk) - self.assertEquals(response.status_code, status.HTTP_204_NO_CONTENT) - - end_count = Datatype.objects.all().count() - self.assertEquals(end_count, start_count - 1) - - -@skipIfDBFeature('is_mocked') -class CompoundDatatypeApiTests(TestCase): - def setUp(self): - self.factory = APIRequestFactory() - self.kive_user = kive_user() - - self.list_path = reverse("compounddatatype-list") - self.detail_pk = 3 - self.detail_path = reverse("compounddatatype-detail", - kwargs={'pk': self.detail_pk}) - self.removal_path = reverse("compounddatatype-removal-plan", - kwargs={'pk': self.detail_pk}) - - # We can't remove the CDT with PK = 3 as it's used by Kive's internals. - self.removal_pk = 4 - - # This should equal metadata.ajax.CompoundDatatypeViewSet.as_view({"get": "list"}). - self.list_view, _, _ = resolve(self.list_path) - self.detail_view, _, _ = resolve(self.detail_path) - self.removal_view, _, _ = resolve(self.removal_path) - - def test_auth(self): - # First try to access while not logged in. - request = self.factory.get(self.list_path) - response = self.list_view(request) - self.assertEquals(response.data["detail"], - "Authentication credentials were not provided.") - - # Now log in and check that "detail" is not passed in the response. - force_authenticate(request, user=self.kive_user) - response = self.list_view(request) - self.assertNotIn('detail', response.data) - - def test_list(self): - """ - Test the CompoundDatatype API list view. - """ - request = self.factory.get(self.list_path) - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - # There are four CDTs loaded into the Database by default, and they're - # sorted alphabetically. See the migration where they're defined for - # details. SQLite sorts case-sensitive and PostgreSQL doesn't. - self.assertEquals(len(response.data), 4) - cdt2 = next(cdt for cdt in response.data if cdt['id'] == 2) - self.assertEquals(cdt2['representation'], - '(failed_row: natural number)') - - def test_detail(self): - request = self.factory.get(self.detail_path) - force_authenticate(request, user=self.kive_user) - response = self.detail_view(request, pk=self.detail_pk) - self.assertEquals(response.data['representation'], - '(example: string?, valid: boolean)') - - def test_removal_plan(self): - request = self.factory.get(self.removal_path) - force_authenticate(request, user=self.kive_user) - response = self.removal_view(request, pk=self.removal_pk) - self.assertEquals(response.data['CompoundDatatypes'], 1) - - def test_removal(self): - start_count = CompoundDatatype.objects.all().count() - - request = self.factory.delete(self.detail_path) - force_authenticate(request, user=self.kive_user) - response = self.detail_view(request, pk=self.removal_pk) - self.assertEquals(response.status_code, status.HTTP_204_NO_CONTENT) - - end_count = CompoundDatatype.objects.all().count() - self.assertEquals(end_count, start_count - 1) - - def test_removal_plan_protected_CDT(self): - """ - Removal plan will not attempt to remove a "protected" CDT (one that's used by Kive's internals). - """ - request = self.factory.get(self.removal_path) - force_authenticate(request, user=self.kive_user) - response = self.removal_view(request, pk=self.detail_pk) - self.assertEquals(response.data['CompoundDatatypes'], 0) - - def test_create(self): - """ - Test creation of a CompoundDatatype using the API. - """ - self.string_dt = Datatype.objects.get(pk=datatypes.STR_PK) - cdt_dict = { - "name": "GoodCDT", - "members": [ - { - "column_idx": 1, - "column_name": "col1", - "datatype": self.string_dt.pk - }, - { - "column_idx": 2, - "column_name": "col2", - "datatype": self.string_dt.pk - } - ], - "groups_allowed": [everyone_group().name] - } - - request = self.factory.post(self.list_path, cdt_dict, format="json") - force_authenticate(request, user=self.kive_user) - self.list_view(request) - - # Probe the resulting method. - cdt = CompoundDatatype.objects.get(name=cdt_dict["name"]) +from django.test import TestCase, skipIfDBFeature +from django.contrib.auth.models import User, Group - self.assertEqual(cdt.name, cdt_dict["name"]) - self.assertEqual(cdt.user, kive_user()) - self.assertSetEqual(set(cdt.groups_allowed.all()), {everyone_group()}) - self.assertFalse(cdt.users_allowed.exists()) +from metadata.models import BasicConstraint, Datatype, everyone_group +from constants import datatypes, groups - self.assertEqual(cdt.members.count(), 2) - col1 = cdt.members.get(column_idx=1) - self.assertEqual(col1.column_name, cdt_dict["members"][0]["column_name"]) - self.assertEqual(col1.datatype, self.string_dt) - col2 = cdt.members.get(column_idx=2) - self.assertEqual(col2.column_name, cdt_dict["members"][1]["column_name"]) - self.assertEqual(col2.datatype, self.string_dt) +samplecode_path = "../samplecode" @skipIfDBFeature('is_mocked') @@ -2697,219 +109,3 @@ def test_intersect_permissions_querysets_everyone_perm(self): groups_qs=self.groups_to_intersect) self.assertSetEqual(set(self.users_to_intersect), set(users_qs)) self.assertSetEqual(set(self.groups_to_intersect), set(groups_qs)) - - -@skipIfDBFeature('is_mocked') -class CompoundDatatypeSerializerTests(TestCase): - - def setUp(self): - self.string_dt = Datatype.objects.get(pk=datatypes.STR_PK) - self.dt_no_permissions = Datatype( - user=kive_user(), - name="DatatypeNoPermissions", - description="Datatype with no added permissions" - ) - self.dt_no_permissions.save() - self.dt_no_permissions.restricts.add(self.string_dt) - - self.dt_developer_permissions = Datatype( - user=kive_user(), - name="DatatypeDeveloperPermissions", - description="Datatype that developers can access" - ) - self.dt_developer_permissions.save() - self.dt_developer_permissions.restricts.add(self.string_dt) - self.dt_developer_permissions.groups_allowed.add( - Group.objects.get(pk=groups.DEVELOPERS_PK) - ) - - self.kive_context = DuckContext(user=kive_user()) - - def test_validate_empty(self): - """ - Testing validation of a CDT with no members. - """ - cdt_dict = { - "name": "EmptyCDT" - } - cdts = CompoundDatatypeSerializer(data=cdt_dict, context=self.kive_context) - self.assertTrue(cdts.is_valid()) - - def test_validate_no_permissions(self): - """ - Testing validation of a good CDT serialization with no permissions granted. - """ - cdt_dict = { - "name": "GoodCDT", - "members": [ - { - "column_idx": 1, - "column_name": "col1", - "datatype": self.string_dt.pk - }, - { - "column_idx": 2, - "column_name": "col2", - "datatype": self.string_dt.pk - } - ] - } - cdts = CompoundDatatypeSerializer(data=cdt_dict, context=self.kive_context) - self.assertTrue(cdts.is_valid()) - - def test_validate_good_permissions(self): - """ - Testing validation of a good CDT serialization with some permissions granted. - """ - cdt_dict = { - "name": "GoodCDT", - "members": [ - { - "column_idx": 1, - "column_name": "col1", - "datatype": self.string_dt.pk - }, - { - "column_idx": 2, - "column_name": "col2", - "datatype": self.string_dt.pk - } - ], - "users_allowed": [kive_user().username], - "groups_allowed": [everyone_group().name] - } - cdts = CompoundDatatypeSerializer(data=cdt_dict, context=self.kive_context) - self.assertTrue(cdts.is_valid()) - - def test_validate_bad_indices(self): - """ - Validation fails if the indices are not consecutive from 1. - """ - cdt_dict = { - "name": "BadIndicesCDT", - "members": [ - { - "column_idx": 1, - "column_name": "col1", - "datatype": self.string_dt.pk - }, - { - "column_idx": 3, - "column_name": "col3bad", - "datatype": self.string_dt.pk - } - ] - } - cdts = CompoundDatatypeSerializer(data=cdt_dict, context=self.kive_context) - self.assertFalse(cdts.is_valid()) - self.assertListEqual( - cdts.errors["non_field_errors"], - ["Column indices must be consecutive starting from 1"] - ) - - def test_validate_bad_permissions(self): - """ - Validation fails if the CDT permissions exceed those of its members. - """ - new_group = Group(name="Interlopers") - new_group.save() - - new_user = User.objects.create_user("NewUser", password="foo") - - # A Datatype that new_group can access but new_user can't. - dt_no_new_user = Datatype( - user=kive_user(), - name="DatatypeNoNewUser", - description="No new users allowed" - ) - dt_no_new_user.save() - dt_no_new_user.restricts.add(self.string_dt) - dt_no_new_user.groups_allowed.add(new_group) - - # A Datatype that new_user can access but new_group can't. - dt_no_new_group = Datatype( - user=kive_user(), - name="DatatypeNoNewGroup", - description="No new groups allowed" - ) - dt_no_new_group.save() - dt_no_new_group.restricts.add(self.string_dt) - dt_no_new_group.users_allowed.add(new_user) - - cdt_dict = { - "name": "BadPermissionsCDT", - "members": [ - { - "column_idx": 1, - "column_name": "col1", - "datatype": self.dt_no_permissions.pk - }, - { - "column_idx": 2, - "column_name": "col2", - "datatype": self.string_dt.pk - }, - { - "column_idx": 3, - "column_name": "col3", - "datatype": dt_no_new_user.pk - }, - { - "column_idx": 4, - "column_name": "col4", - "datatype": dt_no_new_group.pk - } - ], - "users_allowed": [new_user.username], - "groups_allowed": [new_group.name] - } - - cdts = CompoundDatatypeSerializer(data=cdt_dict, context=self.kive_context) - self.assertFalse(cdts.is_valid()) - self.assertListEqual( - cdts.errors["non_field_errors"], - [ - "User {} cannot be granted access".format(new_user.username), - "Group {} cannot be granted access".format(new_group.name) - ] - ) - - def test_create(self): - """ - Test creation of a CompoundDatatype via deserialization. - """ - cdt_dict = { - "name": "GoodCDT", - "members": [ - { - "column_idx": 1, - "column_name": "col1", - "datatype": self.string_dt.pk - }, - { - "column_idx": 2, - "column_name": "col2", - "datatype": self.string_dt.pk - } - ], - "groups_allowed": [everyone_group().name] - } - - cdts = CompoundDatatypeSerializer(data=cdt_dict, context=self.kive_context) - self.assertTrue(cdts.is_valid()) - cdt = cdts.save() - - self.assertEqual(cdt.name, cdt_dict["name"]) - self.assertEqual(cdt.user, kive_user()) - self.assertSetEqual(set(cdt.groups_allowed.all()), {everyone_group()}) - self.assertFalse(cdt.users_allowed.exists()) - - self.assertEqual(cdt.members.count(), 2) - col1 = cdt.members.get(column_idx=1) - self.assertEqual(col1.column_name, cdt_dict["members"][0]["column_name"]) - self.assertEqual(col1.datatype, self.string_dt) - - col2 = cdt.members.get(column_idx=2) - self.assertEqual(col2.column_name, cdt_dict["members"][1]["column_name"]) - self.assertEqual(col2.datatype, self.string_dt) - diff --git a/kive/method/tests.py b/kive/method/tests.py deleted file mode 100644 index b4338a29e..000000000 --- a/kive/method/tests.py +++ /dev/null @@ -1,1768 +0,0 @@ -""" -Unit tests for Shipyard method models. -""" - -import filecmp -import hashlib -import os.path -import shutil -import tempfile -import copy -import re - -from django.conf import settings -from django.contrib.auth.models import User -from django.core.exceptions import ValidationError -from django.core.files import File -from django.core.files.base import ContentFile -from django.core.urlresolvers import resolve -from django.db import transaction -import django.utils.six as dsix - -from django.test import TestCase, skipIfDBFeature -from django_mock_queries.mocks import mocked_relations -from rest_framework import status -from rest_framework.reverse import reverse -from rest_framework.test import force_authenticate - -from constants import datatypes -from kive.tests import BaseTestCases, install_fixture_files -import librarian.models -from metadata.models import CompoundDatatype, Datatype, everyone_group, kive_user -import metadata.tests -from method.ajax import MethodViewSet, MethodFamilyViewSet, DockerImageViewSet -from method.models import CodeResource, CodeResourceRevision, \ - Method, MethodFamily, MethodDependency, DockerImage -from method.serializers import CodeResourceRevisionSerializer, MethodSerializer -import kive.testing_utils as tools -from fleet.workers import Manager - - -# This was previously defined here but has been moved to metadata.tests. -samplecode_path = metadata.tests.samplecode_path - - -@skipIfDBFeature('is_mocked') -class FileAccessTests(TestCase): - serialized_rollback = True - - def setUp(self): - tools.fd_count("FDs (start)") - - # A typical user. - self.user_randy = User.objects.create_user("Randy", "theotherrford@deco.ca", "hat") - self.user_randy.save() - self.user_randy.groups.add(everyone_group()) - self.user_randy.save() - - # Define comp_cr - self.test_cr = CodeResource( - name="Test CodeResource", - description="A test CodeResource to play with file access", - filename="complement.py", - user=self.user_randy) - self.test_cr.save() - - # Define compv1_crRev for comp_cr - self.fn = "complement.py" - - def tearDown(self): - tools.clean_up_all_files() - tools.fd_count("FDs (end)") - - def test_close_save(self): - with open(os.path.join(samplecode_path, self.fn), "rb") as f: - tools.fd_count("!close->save") - - test_crr = CodeResourceRevision( - coderesource=self.test_cr, - revision_name="v1", - revision_desc="First version", - content_file=File(f), - user=self.user_randy) - - with transaction.atomic(): - self.assertRaises(ValueError, test_crr.save) - - def test_access_close_save(self): - with open(os.path.join(samplecode_path, self.fn), "rb") as f: - test_crr = CodeResourceRevision( - coderesource=self.test_cr, - revision_name="v1", - revision_desc="First version", - content_file=File(f), - user=self.user_randy) - - tools.fd_count("!access->close->save") - test_crr.content_file.read() - tools.fd_count("access-!>close->save") - tools.fd_count("access->close-!>save") - with transaction.atomic(): - self.assertRaises(ValueError, test_crr.save) - tools.fd_count("access->close->save!") - - def test_close_access_save(self): - with open(os.path.join(samplecode_path, self.fn), "rb") as f: - test_crr = CodeResourceRevision( - coderesource=self.test_cr, - revision_name="v1", - revision_desc="First version", - content_file=File(f), - user=self.user_randy) - - with transaction.atomic(): - self.assertRaises(ValueError, test_crr.content_file.read) - self.assertRaises(ValueError, test_crr.save) - - def test_save_close_access(self): - with open(os.path.join(samplecode_path, self.fn), "rb") as f: - test_crr = CodeResourceRevision( - coderesource=self.test_cr, - revision_name="v1", - revision_desc="First version", - content_file=File(f), - user=self.user_randy) - test_crr.save() - - test_crr.content_file.read() - tools.fd_count("save->close->access") - - def test_save_close_access_close(self): - with open(os.path.join(samplecode_path, self.fn), "rb") as f: - tools.fd_count("open-!>File->save->close->access->close") - test_crr = CodeResourceRevision( - coderesource=self.test_cr, - revision_name="v1", - revision_desc="First version", - content_file=File(f), - user=self.user_randy) - tools.fd_count("open->File-!>save->close->access->close") - test_crr.save() - tools.fd_count("open->File->save-!>close->access->close") - - tools.fd_count("open->File->save->close-!>access->close") - test_crr.content_file.read() - tools.fd_count("open->File->save->close->access-!>close") - test_crr.content_file.close() - tools.fd_count("open->File->save->close->access->close!") - - def test_save_close_clean_close(self): - with open(os.path.join(samplecode_path, self.fn), "rb") as f: - # Compute the reference MD5 - md5gen = hashlib.md5() - md5gen.update(f.read()) - f_checksum = md5gen.hexdigest() - f.seek(0) - - tools.fd_count("open-!>File->save->close->clean->close") - test_crr = CodeResourceRevision( - coderesource=self.test_cr, - revision_name="v1", - revision_desc="First version", - content_file=File(f), - MD5_checksum=f_checksum, - user=self.user_randy) - - tools.fd_count("open->File-!>save->close->clean->close") - test_crr.save() - tools.fd_count("open->File->save-!>close->clean->close") - - tools.fd_count("open->File->save->close-!>clean->close") - test_crr.clean() - tools.fd_count("open->File->save->close->clean-!>close") - test_crr.content_file.close() - tools.fd_count("open->File->save->close->clean->close!") - - def test_clean_save_close(self): - with open(os.path.join(samplecode_path, self.fn), "rb") as f: - tools.fd_count("open-!>File->clean->save->close") - test_crr = CodeResourceRevision( - coderesource=self.test_cr, - revision_name="v1", - revision_desc="First version", - content_file=File(f), - user=self.user_randy) - tools.fd_count("open->File-!>clean->save->close") - test_crr.clean() - tools.fd_count("open->File->clean-!>save->close") - test_crr.save() - tools.fd_count("open->File->clean->save-!>close") - tools.fd_count("open->File->clean->save->close!") - - def test_clean_save_close_clean_close(self): - with open(os.path.join(samplecode_path, self.fn), "rb") as f: - - tools.fd_count("open-!>File->clean->save->close->clean->close") - test_crr = CodeResourceRevision( - coderesource=self.test_cr, - revision_name="v1", - revision_desc="First version", - content_file=File(f), - user=self.user_randy) - tools.fd_count("open->File-!>clean->save->close->clean->close") - tools.fd_count_logger.debug("FieldFile is open: {}".format(not test_crr.content_file.closed)) - test_crr.clean() - tools.fd_count("open->File->clean-!>save->close->clean->close") - tools.fd_count_logger.debug("FieldFile is open: {}".format(not test_crr.content_file.closed)) - test_crr.save() - tools.fd_count("open->File->clean->save-!>close->clean->close") - tools.fd_count_logger.debug("FieldFile is open: {}".format(not test_crr.content_file.closed)) - - tools.fd_count("open->File->clean->save->close-!>clean->close") - tools.fd_count_logger.debug("FieldFile is open: {}".format(not test_crr.content_file.closed)) - test_crr.clean() - tools.fd_count("open->File->clean->save->close->clean-!>close") - tools.fd_count_logger.debug("FieldFile is open: {}".format(not test_crr.content_file.closed)) - test_crr.content_file.close() - tools.fd_count("open->File->clean->save->close->clean->close!") - tools.fd_count_logger.debug("FieldFile is open: {}".format(not test_crr.content_file.closed)) - - -@skipIfDBFeature('is_mocked') -class MethodTestCase(TestCase, object): - """ - Set up a database state for unit testing. - - This sets up all the stuff used in the Metadata tests, as well as some of the Datatypes - and CDTs we use here. - """ - def setUp(self): - """Set up default database state for Method unit testing.""" - tools.create_method_test_environment(self) - - def tearDown(self): - tools.destroy_method_test_environment(self) - - -class CodeResourceTests(MethodTestCase): - - def setUp(self): - super(CodeResourceTests, self).setUp() - self.cr_filename_err_msg = ('Filename must contain only: alphanumeric characters; spaces; ' - 'and the characters -._(), ' - 'and cannot start with a space') - - def test_str(self): - """ - String representation is the codeResource name. - """ - self.assertEquals(str(self.comp_cr), "complement") - - def test_valid_name_clean_good(self): - """ - Clean passes when codeResource name is file-system valid - """ - valid_cr = CodeResource(name="name", filename="validName", description="desc", user=self.myUser) - valid_cr.save() - self.assertIsNone(valid_cr.clean()) - - def test_valid_name_with_special_symbols_clean_good(self): - """ - Clean passes when codeResource name is file-system valid - """ - valid_cr = CodeResource(name="anotherName", filename="valid.Name with-spaces_and_underscores().py", - description="desc", user=self.myUser) - valid_cr.save() - self.assertIsNone(valid_cr.clean()) - - def test_invalid_name_doubledot_clean_bad(self): - """ - Clean fails when CodeResource name isn't file-system valid - """ - invalid_cr = CodeResource(name="test", filename="../test.py", description="desc", user=self.myUser) - invalid_cr.save() - self.assertRaisesRegexp( - ValidationError, - re.escape(self.cr_filename_err_msg), - invalid_cr.clean_fields - ) - - def test_invalid_name_starting_space_clean_bad(self): - """ - Clean fails when CodeResource name isn't file-system valid - """ - invalid_cr = CodeResource(name="test", filename=" test.py", description="desc", user=self.myUser) - invalid_cr.save() - self.assertRaisesRegexp( - ValidationError, - re.escape(self.cr_filename_err_msg), - invalid_cr.clean_fields - ) - - def test_invalid_name_invalid_symbol_clean_bad(self): - """ - Clean fails when CodeResource name isn't file-system valid - """ - invalid_cr = CodeResource(name="name", filename="test$.py", description="desc", user=self.myUser) - invalid_cr.save() - self.assertRaisesRegexp( - ValidationError, - re.escape(self.cr_filename_err_msg), - invalid_cr.clean_fields - ) - - def test_invalid_name_trailing_space_clean_bad(self): - """ - Clean fails when CodeResource name isn't file-system valid - """ - invalid_cr = CodeResource(name="name", filename="test.py ", description="desc", user=self.myUser) - invalid_cr.save() - self.assertRaisesRegexp( - ValidationError, - re.escape(self.cr_filename_err_msg), - invalid_cr.clean_fields - ) - - -class CodeResourceRevisionTests(MethodTestCase): - - def test_str(self): - """ - String representation of a CodeResourceRevision is its code resource - revision name. - - Or, if no CodeResource has been linked, should display a placeholder. - """ - # Valid crRev should return it's cr.name and crRev.revision_name - self.assertEquals(str(self.compv1_crRev), "complement:1 (v1)") - - # Define a crRev without a linking cr, or a revision_name - no_cr_set = CodeResourceRevision() - self.assertEquals(str(no_cr_set), "[no code resource name]:[no revision number] ([no revision name])") - - # Define a crRev without a linking cr, with a revision_name of foo - no_cr_set.revision_name = "foo" - self.assertEquals(str(no_cr_set), "[no code resource name]:[no revision number] (foo)") - - def test_clean_valid_MD5(self): - """ - An MD5 should exist. - """ - # Compute the reference MD5 - md5gen = hashlib.md5() - with open(os.path.join(samplecode_path, "complement.py"), "rb") as f: - md5gen.update(f.read()) - - # Revision should have the correct MD5 checksum - self.assertEquals(md5gen.hexdigest(), self.comp_cr.revisions.get(revision_name="v1").MD5_checksum) - - def test_find_update_not_found(self): - update = self.compv2_crRev.find_update() - - self.assertEqual(update, None) - - def test_find_update(self): - update = self.compv1_crRev.find_update() - - self.assertEqual(update, self.compv2_crRev) - - -class MethodDependencyTests(MethodTestCase): - def setUp(self): - super(MethodDependencyTests, self).setUp() - - # self.DNAcompv1_m is defined in the setup, based on v1 of self.comp_cr: - # family name: "DNAcomplement" - # revision_name: "v1" - # revision_desc: "First version" - # driver: v1 of self.comp_cr - # v2 is a revision of comp_cr such that revision_name = v2. - self.v2 = self.comp_cr.revisions.get(revision_name="v2") - - def test_str(self): - """ - String representation of a of MethodDependency should look like: - requires as - """ - # Define a fake dependency where the method requires v2 in subdir/foo.py - test_dep = MethodDependency(method=self.DNAcompv1_m, - requirement=self.v2, - path="subdir", - filename="foo.py") - - # Display unicode for this dependency under valid conditions - self.assertEquals( - str(test_dep), - "DNAcomplement DNAcomplement:1 (v1) requires complement complement:2 (v2) as subdir/foo.py" - ) - - def test_invalid_dotdot_path_clean(self): - """ - Dependency tries to go into a path outside its sandbox. - """ - bad_dep = MethodDependency(method=self.DNAcompv1_m, - requirement=self.v2, - path="..", - filename="foo.py") - self.assertRaisesRegexp( - ValidationError, - "path cannot reference \.\./", - bad_dep.clean) - - bad_dep_2 = MethodDependency(method=self.DNAcompv1_m, - requirement=self.v2, - path="../test", - filename="foo.py") - self.assertRaisesRegexp( - ValidationError, - "path cannot reference \.\./", - bad_dep_2.clean) - - def test_valid_path_with_dotdot_clean(self): - """ - Dependency goes into a path with a directory containing ".." in the name. - """ - v2 = self.comp_cr.revisions.get(revision_name="v2") - - good_md = MethodDependency( - method=self.DNAcompv1_m, - requirement=v2, - path="..bar", - filename="foo.py" - ) - self.assertEquals(good_md.clean(), None) - - good_md_2 = MethodDependency( - method=self.DNAcompv1_m, - requirement=v2, - path="bar..", - filename="foo.py" - ) - self.assertEquals(good_md_2.clean(), None) - - good_md_3 = MethodDependency( - method=self.DNAcompv1_m, - requirement=v2, - path="baz/bar..", - filename="foo.py" - ) - self.assertEquals(good_md_3.clean(), None) - - good_md_4 = MethodDependency( - method=self.DNAcompv1_m, - requirement=v2, - path="baz/..bar", - filename="foo.py" - ) - self.assertEquals(good_md_4.clean(), None) - - good_md_5 = MethodDependency( - method=self.DNAcompv1_m, - requirement=v2, - path="baz/..bar..", - filename="foo.py" - ) - self.assertEquals(good_md_5.clean(), None) - - good_md_6 = MethodDependency( - method=self.DNAcompv1_m, - requirement=v2, - path="..baz/bar..", - filename="foo.py" - ) - self.assertEquals(good_md_6.clean(), None) - - # This case works because the ".." doesn't take us out of the sandbox - good_md_7 = MethodDependency( - method=self.DNAcompv1_m, - requirement=v2, - path="baz/../bar", - filename="foo.py" - ) - self.assertEquals(good_md_7.clean(), None) - - good_md_8 = MethodDependency( - method=self.DNAcompv1_m, - requirement=v2, - path="baz/..bar../blah", - filename="foo.py") - self.assertEquals(good_md_8.clean(), None) - - def test_method_dependency_with_good_path_and_filename_clean(self): - """ - Test a MethodDependency with no problems. - """ - v2 = self.comp_cr.revisions.get(revision_name="v2") - - # Define a MethodDependency for self.DNAcompv1_m with good paths and filenames. - good_md = MethodDependency( - method=self.DNAcompv1_m, - requirement=v2, - path="testFolder/anotherFolder", - filename="foo.py") - self.assertEqual(good_md.clean(), None) - - -class MethodInstallTests(MethodTestCase): - """Tests of the install function of Method.""" - def setUp(self): - super(MethodInstallTests, self).setUp() - - # This method is defined in testing_utils.create_method_test_environment. - # It has self.compv1_crRev as its driver, and no dependencies. - self.independent_method = self.DNAcompv1_m - - # This method is defined in testing_utils.create_method_test_environment. - # It has self.compv2_crRev as its driver and dna_resource_revision (not - # bound to self) as a dependency. - self.dependant_method = self.DNAcompv2_m - - def test_base_case(self): - """ - Test of base case -- installing a Method with no dependencies. - """ - test_path = tempfile.mkdtemp(prefix="test_base_case") - - self.independent_method.install(test_path) - self.assertTrue(os.path.exists(os.path.join(test_path, "complement.py"))) - - shutil.rmtree(test_path) - - def test_second_revision(self): - """ - Test of base case -- installing a Method that is a second revision. - - This Method does have a dependency. - """ - test_path = tempfile.mkdtemp(prefix="test_base_case") - - self.dependant_method.install(test_path) - self.assertTrue(os.path.exists(os.path.join(test_path, "complement.py"))) - self.assertTrue(os.path.exists(os.path.join(test_path, "good_dna.csv"))) - - shutil.rmtree(test_path) - - def test_dependency_same_dir_dot(self): - """ - Test of installing a Method with a dependency in the same directory, specified using a dot. - """ - test_path = tempfile.mkdtemp(prefix="test_dependency_same_dir_dot") - - self.independent_method.dependencies.create(requirement=self.test_cr_1_rev1, path=".") - self.independent_method.install(test_path) - self.assertTrue(os.path.exists(os.path.join(test_path, "complement.py"))) - self.assertTrue(os.path.exists(os.path.join(test_path, "test_cr_1.py"))) - - shutil.rmtree(test_path) - - def test_dependency_same_dir_blank(self): - """ - Test of installing a Method with a dependency in the same directory, specified using a blank. - """ - test_path = tempfile.mkdtemp(prefix="test_dependency_same_dir_blank") - - self.independent_method.dependencies.create(requirement=self.test_cr_1_rev1, path="") - self.independent_method.install(test_path) - self.assertTrue(os.path.exists(os.path.join(test_path, "complement.py"))) - self.assertTrue(os.path.exists(os.path.join(test_path, "test_cr_1.py"))) - - shutil.rmtree(test_path) - - def test_dependency_override_dep_filename(self): - """ - Test of installing a Method with a dependency whose filename is overridden. - """ - test_path = tempfile.mkdtemp(prefix="test_dependency_override_dep_filename") - - self.independent_method.dependencies.create(requirement=self.test_cr_1_rev1, path="", - filename="foo.py") - self.independent_method.install(test_path) - self.assertTrue(os.path.exists(os.path.join(test_path, "complement.py"))) - self.assertTrue(os.path.exists(os.path.join(test_path, "foo.py"))) - self.assertFalse(os.path.exists(os.path.join(test_path, "test_cr_1.py"))) - - shutil.rmtree(test_path) - - def test_dependency_in_subdirectory(self): - """ - Test of installing a Method with a dependency in a subdirectory. - """ - test_path = tempfile.mkdtemp(prefix="test_dependency_in_subdirectory") - - self.independent_method.dependencies.create(requirement=self.test_cr_1_rev1, path="modules") - self.independent_method.install(test_path) - self.assertTrue(os.path.exists(os.path.join(test_path, "complement.py"))) - self.assertTrue(os.path.isdir(os.path.join(test_path, "modules"))) - self.assertTrue(os.path.exists(os.path.join(test_path, "modules", "test_cr_1.py"))) - - shutil.rmtree(test_path) - - def test_dependencies_in_same_subdirectory(self): - """ - Test of installing a Method with several dependencies in the same subdirectory. - """ - test_path = tempfile.mkdtemp(prefix="test_dependencies_in_same_subdirectory") - - self.independent_method.dependencies.create(requirement=self.test_cr_1_rev1, path="modules") - self.independent_method.dependencies.create(requirement=self.test_cr_2_rev1, path="modules") - self.independent_method.install(test_path) - self.assertTrue(os.path.exists(os.path.join(test_path, "complement.py"))) - self.assertTrue(os.path.isdir(os.path.join(test_path, "modules"))) - self.assertTrue(os.path.exists(os.path.join(test_path, "modules", "test_cr_1.py"))) - self.assertTrue(os.path.exists(os.path.join(test_path, "modules", "test_cr_2.py"))) - - shutil.rmtree(test_path) - - def test_dependencies_in_same_directory(self): - """ - Test of installing a Method with several dependencies in the base directory. - """ - test_path = tempfile.mkdtemp(prefix="test_dependencies_in_same_directory") - - self.independent_method.dependencies.create(requirement=self.test_cr_1_rev1, path="") - self.independent_method.dependencies.create(requirement=self.test_cr_2_rev1, path="") - self.independent_method.install(test_path) - self.assertTrue(os.path.exists(os.path.join(test_path, "complement.py"))) - self.assertTrue(os.path.exists(os.path.join(test_path, "test_cr_1.py"))) - self.assertTrue(os.path.exists(os.path.join(test_path, "test_cr_2.py"))) - - shutil.rmtree(test_path) - - def test_dependencies_in_subsub_directory(self): - """ - Test of installing a Method with dependencies in sub-sub-directories. - """ - test_path = tempfile.mkdtemp(prefix="test_dependencies_in_subsub_directory") - - self.independent_method.dependencies.create(requirement=self.test_cr_1_rev1, path="modules/foo1") - self.independent_method.dependencies.create(requirement=self.test_cr_2_rev1, path="modules/foo2") - self.independent_method.install(test_path) - self.assertTrue(os.path.exists(os.path.join(test_path, "complement.py"))) - self.assertTrue(os.path.isdir(os.path.join(test_path, "modules/foo1"))) - self.assertTrue(os.path.isdir(os.path.join(test_path, "modules/foo2"))) - self.assertTrue(os.path.exists(os.path.join(test_path, "modules", "foo1", "test_cr_1.py"))) - self.assertTrue(os.path.exists(os.path.join(test_path, "modules", "foo2", "test_cr_2.py"))) - - shutil.rmtree(test_path) - - def test_dependencies_from_same_coderesource_same_dir(self): - """ - Test of installing a Method with a dependency having the same CodeResource in the same directory. - """ - test_path = tempfile.mkdtemp(prefix="test_dependencies_from_same_coderesource_same_dir") - - self.independent_method.dependencies.create(requirement=self.compv2_crRev, path="", filename="foo.py") - self.independent_method.install(test_path) - self.assertTrue(os.path.exists(os.path.join(test_path, "complement.py"))) - self.assertTrue(os.path.exists(os.path.join(test_path, "foo.py"))) - # Test that the right files are in the right places. - self.assertTrue( - filecmp.cmp(os.path.join(samplecode_path, "complement.py"), - os.path.join(test_path, "complement.py")) - ) - self.assertTrue( - filecmp.cmp(os.path.join(samplecode_path, "complement_v2.py"), - os.path.join(test_path, "foo.py")) - ) - - shutil.rmtree(test_path) - - def test_dependencies_in_various_places(self): - """ - Test of installing a Method with dependencies in several places. - """ - test_path = tempfile.mkdtemp(prefix="test_dependencies_in_various_places") - - self.independent_method.dependencies.create(requirement=self.test_cr_1_rev1, path="modules") - self.independent_method.dependencies.create(requirement=self.test_cr_2_rev1, path="moremodules") - self.independent_method.dependencies.create(requirement=self.test_cr_3_rev1, path="modules/foo") - self.independent_method.install(test_path) - self.assertTrue(os.path.exists(os.path.join(test_path, "complement.py"))) - self.assertTrue(os.path.isdir(os.path.join(test_path, "modules"))) - self.assertTrue(os.path.isdir(os.path.join(test_path, "moremodules"))) - self.assertTrue(os.path.isdir(os.path.join(test_path, "modules", "foo"))) - self.assertTrue(os.path.exists(os.path.join(test_path, "modules", "test_cr_1.py"))) - self.assertTrue(os.path.exists(os.path.join(test_path, "moremodules", "test_cr_2.py"))) - self.assertTrue(os.path.exists(os.path.join(test_path, "modules", "foo", "test_cr_3.py"))) - - shutil.rmtree(test_path) - - -class MethodTests(MethodTestCase): - - def setUp(self): - super(MethodTests, self).setUp() - self.test_dep_method = tools.make_first_method( - "TestMethodDependencies", - "Methods with dependencies", - self.test_cr_1_rev1, - self.myUser - ) - tools.simple_method_io(self.test_dep_method, None, "dummy_in", "dummy_out") - - def test_delete_method(self): - """Deleting a method is possible.""" - Method.objects.first().delete() - - def test_create(self): - """Create a new Method by the constructor.""" - names = ["a", "b"] - cdts = CompoundDatatype.objects.all()[:2] - family = MethodFamily.objects.first() - driver = CodeResourceRevision.objects.first() - m = Method.create(names, compounddatatypes=cdts, num_inputs=1, family=family, driver=driver, user=self.myUser) - self.assertIsNone(m.complete_clean()) - - -class MethodFamilyTests(MethodTestCase): - - def test_str(self): - """ - String representation of a MethodFamily is its name. - """ - self.assertEqual(str(self.DNAcomp_mf), "DNAcomplement") - - -@skipIfDBFeature('is_mocked') -class NonReusableMethodTests(TestCase): - def setUp(self): - # An unpredictable, non-reusable user. - self.user_rob = User.objects.create_user('rob', 'rford@toronto.ca', 'football') - self.user_rob.save() - self.user_rob.groups.add(everyone_group()) - self.user_rob.save() - - # A piece of code that is non-reusable. - self.rng = tools.make_first_revision( - "rng", "Generates a random number", "rng.py", - """#! /usr/bin/env python - -import random -import csv -import sys - -outfile = sys.argv[1] - -with open(outfile, "wb") as f: - my_writer = csv.writer(f) - my_writer.writerow(("random number",)) - my_writer.writerow((random.random(),)) -""", - self.user_rob - ) - - self.rng_out_cdt = CompoundDatatype(user=self.user_rob) - self.rng_out_cdt.save() - self.rng_out_cdt.members.create( - column_name="random number", column_idx=1, - datatype=Datatype.objects.get(pk=datatypes.FLOAT_PK) - ) - self.rng_out_cdt.grant_everyone_access() - - self.rng_method = tools.make_first_method("rng", "Generate a random number", self.rng, - self.user_rob) - self.rng_method.create_output(dataset_name="random_number", dataset_idx=1, compounddatatype=self.rng_out_cdt, - min_row=1, max_row=1) - self.rng_method.reusable = Method.NON_REUSABLE - self.rng_method.save() - - self.increment = tools.make_first_revision( - "increment", "Increments all numbers in its first input file by the number in its second", - "increment.py", - """#! /usr/bin/env python - -import csv -import sys - -numbers_file = sys.argv[1] -increment_file = sys.argv[2] -outfile = sys.argv[3] - -incrementor = 0 -with open(increment_file, "rb") as f: - inc_reader = csv.DictReader(f) - for row in inc_reader: - incrementor = float(row["incrementor"]) - break - -numbers = [] -with open(numbers_file, "rb") as f: - number_reader = csv.DictReader(f) - for row in number_reader: - numbers.append(float(row["number"])) - -with open(outfile, "wb") as f: - out_writer = csv.writer(f) - out_writer.writerow(("incremented number",)) - for number in numbers: - out_writer.writerow((number + incrementor,)) -""", - self.user_rob - ) - - self.increment_in_1_cdt = CompoundDatatype(user=self.user_rob) - self.increment_in_1_cdt.save() - self.increment_in_1_cdt.members.create( - column_name="number", column_idx=1, - datatype=Datatype.objects.get(pk=datatypes.FLOAT_PK) - ) - self.increment_in_1_cdt.grant_everyone_access() - - self.increment_in_2_cdt = CompoundDatatype(user=self.user_rob) - self.increment_in_2_cdt.save() - self.increment_in_2_cdt.members.create( - column_name="incrementor", column_idx=1, - datatype=Datatype.objects.get(pk=datatypes.FLOAT_PK) - ) - self.increment_in_2_cdt.grant_everyone_access() - - self.increment_out_cdt = CompoundDatatype(user=self.user_rob) - self.increment_out_cdt.save() - self.increment_out_cdt.members.create( - column_name="incremented number", column_idx=1, - datatype=Datatype.objects.get(pk=datatypes.FLOAT_PK) - ) - self.increment_out_cdt.grant_everyone_access() - - self.inc_method = tools.make_first_method( - "increment", "Increments all numbers in its first input file by the number in its second", - self.increment, self.user_rob) - self.inc_method.create_input(dataset_name="numbers", dataset_idx=1, compounddatatype=self.increment_in_1_cdt) - self.inc_method.create_input(dataset_name="incrementor", dataset_idx=2, - compounddatatype=self.increment_in_2_cdt, - min_row=1, max_row=1) - self.inc_method.create_output(dataset_name="incremented_numbers", dataset_idx=1, - compounddatatype=self.increment_out_cdt) - - self.test_nonreusable = tools.make_first_pipeline("Non-Reusable", "Pipeline with a non-reusable step", - self.user_rob) - self.test_nonreusable.create_input(dataset_name="numbers", dataset_idx=1, - compounddatatype=self.increment_in_1_cdt) - self.test_nonreusable.steps.create( - step_num=1, - transformation=self.rng_method, - name="source of randomness" - ) - - step2 = self.test_nonreusable.steps.create( - step_num=2, - transformation=self.inc_method, - name="incrementor" - ) - step2.cables_in.create( - dest=self.inc_method.inputs.get(dataset_name="numbers"), - source_step=0, - source=self.test_nonreusable.inputs.get(dataset_name="numbers") - ) - connecting_cable = step2.cables_in.create( - dest=self.inc_method.inputs.get(dataset_name="incrementor"), - source_step=1, - source=self.rng_method.outputs.get(dataset_name="random_number") - ) - connecting_cable.custom_wires.create( - source_pin=self.rng_out_cdt.members.get(column_name="random number"), - dest_pin=self.increment_in_2_cdt.members.get(column_name="incrementor") - ) - - self.test_nonreusable.create_outcable( - output_name="incremented_numbers", - output_idx=1, - source_step=2, - source=self.inc_method.outputs.get(dataset_name="incremented_numbers") - ) - - self.test_nonreusable.create_outputs() - - # A data file to add to the database. - self.numbers = "number\n1\n2\n3\n4\n" - with tempfile.NamedTemporaryFile(mode="wt", delete=False) as datafile: - datafile.write(self.numbers) - - # Alice uploads the data to the system. - self.numbers_dataset = librarian.models.Dataset.create_dataset( - datafile.name, - user=self.user_rob, - groups_allowed=[everyone_group()], - cdt=self.increment_in_1_cdt, keep_file=True, - name="numbers", description="1-2-3-4" - ) - - def test_find_compatible_ER_non_reusable_method(self): - """ - The ExecRecord of a non-reusable Method should not be found compatible. - """ - Manager.execute_pipeline(self.user_rob, self.test_nonreusable, [self.numbers_dataset]) - - rng_step = self.test_nonreusable.steps.get(step_num=1) - runstep = rng_step.pipelinestep_instances.first() - self.assertListEqual(list(runstep.find_compatible_ERs([])), []) - - -class MethodFamilyApiMockTests(BaseTestCases.ApiTestCase): - def setUp(self): - self.mock_viewset(MethodFamilyViewSet) - super(MethodFamilyApiMockTests, self).setUp() - - self.list_path = reverse("methodfamily-list") - self.detail_pk = 43 - self.detail_path = reverse("methodfamily-detail", - kwargs={'pk': self.detail_pk}) - self.removal_path = reverse("methodfamily-removal-plan", - kwargs={'pk': self.detail_pk}) - - self.list_view, _, _ = resolve(self.list_path) - self.detail_view, _, _ = resolve(self.detail_path) - self.removal_view, _, _ = resolve(self.removal_path) - - MethodFamily.objects.add(MethodFamily(pk=42, - user=self.kive_kive_user, - name='mA_name turnip', - description='A_desc'), - MethodFamily(pk=43, - user=self.kive_kive_user, - name='mB_name', - description='B_desc'), - MethodFamily(pk=44, - user=self.kive_kive_user, - name='mC_name', - description='C_desc turnip')) - - def test_list(self): - """ - Test the API list view. - """ - request = self.factory.get(self.list_path) - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - self.assertEquals(len(response.data), 3) - self.assertEquals(response.data[2]['name'], 'mC_name') - - def test_detail(self): - request = self.factory.get(self.detail_path) - force_authenticate(request, user=self.kive_user) - response = self.detail_view(request, pk=self.detail_pk) - self.assertEquals(response.data['name'], 'mB_name') - - def test_removal_plan(self): - request = self.factory.get(self.removal_path) - force_authenticate(request, user=self.kive_user) - response = self.removal_view(request, pk=self.detail_pk) - self.assertEquals(response.data['MethodFamilies'], 1) - - def test_filter_name(self): - request = self.factory.get( - self.list_path + "?filters[0][key]=name&filters[0][val]=turnip") - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - self.assertEquals(len(response.data), 1) - self.assertEquals(response.data[0]['name'], 'mA_name turnip') - - def test_filter_description(self): - request = self.factory.get( - self.list_path + "?filters[0][key]=description&filters[0][val]=B_desc") - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - self.assertEquals(len(response.data), 1) - self.assertEquals(response.data[0]['name'], 'mB_name') - - def test_filter_smart(self): - request = self.factory.get( - self.list_path + "?filters[0][key]=smart&filters[0][val]=turnip") - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - self.assertEquals(len(response.data), 2) - self.assertEquals(response.data[0]['name'], 'mA_name turnip') - self.assertEquals(response.data[1]['description'], 'C_desc turnip') - - def test_filter_user(self): - request = self.factory.get( - self.list_path + "?filters[0][key]=user&filters[0][val]=kive") - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - self.assertEquals(len(response.data), 3) - - def test_filter_unknown(self): - request = self.factory.get( - self.list_path + "?filters[0][key]=bogus&filters[0][val]=kive") - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - self.assertEquals({u'detail': u'Unknown filter key: bogus'}, - response.data) - - -@skipIfDBFeature('is_mocked') -class MethodFamilyApiTests(BaseTestCases.ApiTestCase): - fixtures = ['demo'] - - def setUp(self): - super(MethodFamilyApiTests, self).setUp() - - self.list_path = reverse("methodfamily-list") - self.detail_pk = MethodFamily.objects.get(name='sums_and_products').pk - - # This should equal metadata.ajax.CompoundDatatypeViewSet.as_view({"get": "list"}). - self.list_view, _, _ = resolve(self.list_path) - - def add_dependency(self, family, name): - driver = CodeResourceRevision.objects.first() - method = family.members.create(user=family.user, - driver=driver) - code_resource = CodeResource.objects.create(name=name, - user=family.user) - revision = code_resource.revisions.create(user=family.user) - MethodDependency.objects.create(method=method, requirement=revision) - - def test_filter_code_dependency(self): - target_family = MethodFamily.objects.get(pk=self.detail_pk) - other_family = MethodFamily.objects.exclude(pk=self.detail_pk).first() - self.add_dependency(target_family, 'target_code') - self.add_dependency(other_family, 'other_code') - - request = self.factory.get( - self.list_path + "?filters[0][key]=code&filters[0][val]=target") - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - self.assertEquals(len(response.data), 1) - self.assertEquals(response.data[0]['name'], 'sums_and_products') - - def add_driver(self, family, name): - code_resource = CodeResource.objects.create(name=name, - user=family.user) - driver = code_resource.revisions.create(user=family.user) - family.members.create(user=family.user, driver=driver) - - def test_filter_code_driver(self): - target_family = MethodFamily.objects.get(pk=self.detail_pk) - other_family = MethodFamily.objects.exclude(pk=self.detail_pk).first() - self.add_driver(target_family, 'target_code') - self.add_driver(target_family, 'target2_code') # test dups - self.add_driver(other_family, 'other_code') - - request = self.factory.get( - self.list_path + "?filters[0][key]=code&filters[0][val]=target") - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - self.assertEquals(len(response.data), 1) - self.assertEquals(response.data[0]['name'], 'sums_and_products') - - -@skipIfDBFeature('is_mocked') -class CodeResourceApiTests(BaseTestCases.ApiTestCase): - fixtures = ["removal"] - - def setUp(self): - super(CodeResourceApiTests, self).setUp() - - self.list_path = reverse("coderesource-list") - self.list_view, _, _ = resolve(self.list_path) - - # This user is defined in the removal fixture. - self.remover = User.objects.get(pk=2) - self.noop_cr = CodeResource.objects.get(name="Noop") - - self.detail_path = reverse("coderesource-detail", kwargs={"pk": self.noop_cr.pk}) - self.detail_view, _, _ = resolve(self.detail_path) - self.removal_plan = self.noop_cr.build_removal_plan() - - def test_list_url(self): - """ - Test that the API list URL is correctly defined. - """ - # Check that the URL is correctly defined. - self.assertEquals(self.list_path, "/api/coderesources/") - - def test_list(self): - request = self.factory.get(self.list_path) - force_authenticate(request, user=self.remover) - response = self.list_view(request, pk=None) - - self.assertSetEqual( - set([x["name"] for x in response.data]), - set([x.name for x in CodeResource.objects.filter(user=self.remover)]) - ) - - def test_detail(self): - self.assertEquals(self.detail_path, "/api/coderesources/{}/".format(self.noop_cr.pk)) - - request = self.factory.get(self.detail_path) - force_authenticate(request, user=self.remover) - response = self.detail_view(request, pk=self.noop_cr.pk) - detail = response.data - - self.assertEquals(detail["id"], self.noop_cr.pk) - self.assertEquals(detail["num_revisions"], self.noop_cr.num_revisions) - self.assertRegexpMatches( - detail["absolute_url"], - "/resource_revisions/{}/?".format(self.noop_cr.pk) - ) - - def test_removal_plan(self): - cr_removal_path = reverse("coderesource-removal-plan", kwargs={'pk': self.noop_cr.pk}) - cr_removal_view, _, _ = resolve(cr_removal_path) - - request = self.factory.get(cr_removal_path) - force_authenticate(request, user=self.remover) - response = cr_removal_view(request, pk=self.noop_cr.pk) - - for key in self.removal_plan: - self.assertEquals(response.data[key], len(self.removal_plan[key])) - self.assertEquals(response.data['CodeResources'], 1) - self.assertEquals(response.data["CodeResourceRevisions"], 1) - - def test_removal(self): - start_count = CodeResource.objects.count() - start_crr_count = CodeResourceRevision.objects.count() - - request = self.factory.delete(self.detail_path) - force_authenticate(request, user=self.kive_user) - response = self.detail_view(request, pk=self.noop_cr.pk) - self.assertEquals(response.status_code, status.HTTP_204_NO_CONTENT) - - end_count = CodeResource.objects.count() - end_crr_count = CodeResourceRevision.objects.count() - self.assertEquals(end_count, start_count - len(self.removal_plan["CodeResources"])) - self.assertEquals(end_crr_count, start_crr_count - len(self.removal_plan["CodeResourceRevisions"])) - - def test_revisions(self): - cr_revisions_path = reverse("coderesource-revisions", kwargs={"pk": self.noop_cr.pk}) - cr_revisions_view, _, _ = resolve(cr_revisions_path) - - request = self.factory.get(cr_revisions_path) - force_authenticate(request, user=self.remover) - response = cr_revisions_view(request, pk=self.noop_cr.pk) - - self.assertSetEqual(set([x.revision_number for x in self.noop_cr.revisions.all()]), - set([x["revision_number"] for x in response.data])) - - -def crr_test_setup(case): - """ - A helper for CodeResourceRevisionApiTests and CodeResourceRevisionSerializerTests. - """ - # An innocent bystander. - case.innocent_bystander = User.objects.create_user( - "InnocentBystander", "innocent_bystander_1@aol.net", password="WhoMe?") - - # A mock request that we pass as context to our serializer. - class DuckRequest(object): - pass - - case.duck_request = DuckRequest() - case.duck_request.user = kive_user() - case.duck_context = {"request": case.duck_request} - - case.cr_name = "Deserialization Test Family" - case.cr = CodeResource(name=case.cr_name, - filename="HelloWorld.py", - description="Hello World", - user=kive_user()) - case.cr.save() - case.cr.grant_everyone_access() - - case.hello_world = """#!/bin/bash - -echo "Hello World" -""" - case.hello_world_file = ContentFile(case.hello_world) - - # Note: we have to add content_file on the fly so we can wrap it in a Django File. - case.crr_data = { - "coderesource": case.cr_name, - "revision_name": "v1", - "revision_desc": "First version", - "content_file": case.hello_world_file, - "groups_allowed": [everyone_group().name] - } - - # In many situations below, the ContentFile object doesn't work with the DRF serializers. - # In those situations, we'll open the file in the test and insert it into case.crr_data - # so that we don't have an open file handle being passed around. - file_prefix = os.path.join(settings.MEDIA_ROOT, - CodeResourceRevision.UPLOAD_DIR, - 'hello') - with tempfile.NamedTemporaryFile(mode="w+t", - prefix=file_prefix, - suffix='.sh', - delete=False) as f: - case.hello_world_filename = f.name - f.write(case.hello_world) - - case.crr_dependency = """language = ENG""" - case.crr_dependency_file = ContentFile(case.crr_dependency) - case.crd = CodeResourceRevision( - coderesource=case.cr, - revision_name="dependency", - revision_desc="Dependency", - user=kive_user(), - content_file=case.crr_dependency_file - ) - case.crd.clean() - case.crd.save() - case.crd.grant_everyone_access() - - -@skipIfDBFeature('is_mocked') -class CodeResourceRevisionSerializerTests(TestCase): - fixtures = ["removal"] - - def setUp(self): - install_fixture_files("removal") - # This user is defined in the removal fixture. - self.remover = User.objects.get(username="RemOver") - # noinspection PyTypeChecker - crr_test_setup(self) - - def tearDown(self): - tools.clean_up_all_files() - - # Note: all validation tests are redundant. There is no customized validation code anymore. - def test_validate(self): - """ - Test validation of a CodeResourceRevision with no dependencies. - """ - with open(self.hello_world_filename, "rb") as f: - self.crr_data["content_file"] = File(f) - crr_s = CodeResourceRevisionSerializer( - data=self.crr_data, - context={"request": self.duck_request} - ) - self.assertTrue(crr_s.is_valid()) - - def test_create(self): - """ - Test creation of a CodeResourceRevision with no dependencies. - """ - with open(self.hello_world_filename, "rb") as f: - self.crr_data["content_file"] = File(f) - crr_s = CodeResourceRevisionSerializer( - data=self.crr_data, - context={"request": self.duck_request} - ) - crr_s.is_valid() - crr_s.save() - - # Inspect the revision we just added. - new_crr = self.cr.revisions.get(revision_name="v1") - self.assertEquals(new_crr.revision_desc, "First version") - - -@skipIfDBFeature('is_mocked') -class CodeResourceRevisionApiTests(BaseTestCases.ApiTestCase): - fixtures = ["removal"] - - def setUp(self): - install_fixture_files("removal") - # This user is defined in the removal fixture. - self.remover = User.objects.get(username="RemOver") - super(CodeResourceRevisionApiTests, self).setUp() - - self.list_path = reverse("coderesourcerevision-list") - self.list_view, _, _ = resolve(self.list_path) - - self.noop_cr = CodeResource.objects.get(name="Noop") - self.noop_crr = self.noop_cr.revisions.get(revision_number=1) - - self.detail_path = reverse("coderesourcerevision-detail", kwargs={"pk": self.noop_crr.pk}) - self.detail_view, _, _ = resolve(self.detail_path) - - self.removal_plan = self.noop_crr.build_removal_plan() - - self.download_path = reverse("coderesourcerevision-download", kwargs={"pk": self.noop_crr.pk}) - self.download_view, _, _ = resolve(self.download_path) - - # noinspection PyTypeChecker - crr_test_setup(self) - - def tearDown(self): - tools.clean_up_all_files() - os.remove(self.hello_world_filename) - - def test_list(self): - request = self.factory.get(self.list_path) - force_authenticate(request, user=self.remover) - response = self.list_view(request, pk=None) - - tester = self.assertCountEqual - if dsix.PY2: - tester = self.assertItemsEqual - tester( - [x.pk for x in CodeResourceRevision.filter_by_user(user=self.remover)], - [x["id"] for x in response.data] - ) - - def test_detail(self): - request = self.factory.get(self.detail_path) - force_authenticate(request, user=self.remover) - response = self.detail_view(request, pk=self.noop_crr.pk) - detail = response.data - - self.assertEquals(detail["id"], self.noop_crr.pk) - self.assertRegexpMatches( - detail["absolute_url"], - "/resource_revision_add/{}/?".format(self.noop_crr.pk) - ) - self.assertEquals(detail["revision_name"], self.noop_crr.revision_name) - - def test_removal_plan(self): - crr_removal_path = reverse("coderesourcerevision-removal-plan", kwargs={'pk': self.noop_crr.pk}) - crr_removal_view, _, _ = resolve(crr_removal_path) - - request = self.factory.get(crr_removal_path) - force_authenticate(request, user=self.remover) - response = crr_removal_view(request, pk=self.noop_crr.pk) - - for key in self.removal_plan: - self.assertEquals(response.data[key], len(self.removal_plan[key])) - self.assertEquals(response.data["CodeResourceRevisions"], 1) - - def test_removal(self): - start_count = CodeResourceRevision.objects.count() - - request = self.factory.delete(self.detail_path) - force_authenticate(request, user=self.kive_user) - response = self.detail_view(request, pk=self.noop_crr.pk) - self.assertEquals(response.status_code, status.HTTP_204_NO_CONTENT) - - end_count = CodeResourceRevision.objects.count() - # In the above we confirmed this length is 1. - self.assertEquals(end_count, start_count - len(self.removal_plan["CodeResourceRevisions"])) - - def test_create(self): - """ - Test creation of a new CodeResourceRevision via the API. - """ - with open(self.hello_world_filename, "rb") as f: - self.crr_data["content_file"] = f - request = self.factory.post(self.list_path, self.crr_data) - force_authenticate(request, user=kive_user()) - self.list_view(request) - - # Inspect the revision we just added. - new_crr = self.cr.revisions.get(revision_name="v1") - self.assertEquals(new_crr.revision_desc, "First version") - - def test_create_clean_fails(self): - """ - Test that clean is being called during creation. - """ - # Disallow everyone from accessing self.cr, which will cause clean to fail. - self.cr.groups_allowed.remove(everyone_group()) - - with open(self.hello_world_filename, "rb") as f: - self.crr_data["content_file"] = f - request = self.factory.post(self.list_path, self.crr_data) - force_authenticate(request, user=kive_user()) - response = self.list_view(request) - - self.assertDictEqual( - response.data, - {'non_field_errors': "Group(s) Everyone cannot be granted access"} - ) - - def test_download(self): - request = self.factory.get(self.download_path) - force_authenticate(request, user=self.remover) - response = self.download_view(request, pk=self.noop_crr.pk) - - self.assertIn("Content-Disposition", response) - self.assertTrue(response["Content-Disposition"].startswith("attachment; filename=")) - - -def method_test_setup(case): - """ - Helper to set up MethodSerializerTests and MethodApiTests. - """ - crr_test_setup(case) - - # We need a CodeResourceRevision to create a Method from. - with open(case.hello_world_filename, "rb") as f: - case.crr_data["content_file"] = File(f) - crr_s = CodeResourceRevisionSerializer(data=case.crr_data, context=case.duck_context) - crr_s.is_valid() - case.crr = crr_s.save() - - case.docker_image = DockerImage.objects.create(name='foo', - tag='v1.0', - git='http://server/x/y.git', - user=kive_user()) - case.docker_image_url = reverse('dockerimage-detail', - kwargs=dict(pk=case.docker_image.pk)) - - # We need a MethodFamily to add the Method to. - case.dtf_mf = MethodFamily( - name="Deserialization Test Family Methods", - description="For testing the Method serializer.", - user=kive_user() - ) - case.dtf_mf.save() - case.dtf_mf.users_allowed.add(case.innocent_bystander) - case.dtf_mf.grant_everyone_access() - case.dtf_mf.save() - - case.method_data = { - "family": case.dtf_mf.name, - "revision_name": "v1", - "revision_desc": "First version", - "users_allowed": [case.innocent_bystander.username], - "groups_allowed": [everyone_group().name], - "driver": case.crr.pk, - "docker_image": case.docker_image_url, - "inputs": [ - { - "dataset_name": "ignored_input", - "dataset_idx": 1, - "x": 0.1, - "y": 0.1 - }, - { - "dataset_name": "another_ignored_input", - "dataset_idx": 2, - "x": 0.1, - "y": 0.2 - } - ], - "outputs": [ - { - "dataset_name": "empty_output", - "dataset_idx": 1 - } - ] - } - - case.method_data_with_dep = copy.deepcopy(case.method_data) - case.method_data_with_dep["revision_name"] = "v2" - case.method_data_with_dep["revision_desc"] = "Has dependencies" - case.method_data_with_dep["dependencies"] = [ - { - "requirement": case.crd.pk, - "filename": "config.dat" - }, - { - "requirement": case.crd.pk, - "path": "configuration.dat", - "filename": "config_2.dat" - } - ] - - -@skipIfDBFeature('is_mocked') -class MethodSerializerTests(TestCase): - fixtures = ["removal"] - - def setUp(self): - install_fixture_files("removal") - # noinspection PyTypeChecker - method_test_setup(self) - - def tearDown(self): - tools.clean_up_all_files() - - def test_create(self): - """ - Test creation of a Method using the serializer. - """ - method_s = MethodSerializer(data=self.method_data, context=self.duck_context) - self.assertTrue(method_s.is_valid()) - new_method = method_s.save() - - # Probe the new method to see that it got created correctly. - self.assertEquals(new_method.inputs.count(), 2) - in_1 = new_method.inputs.get(dataset_idx=1) - in_2 = new_method.inputs.get(dataset_idx=2) - - self.assertEquals(in_1.dataset_name, "ignored_input") - self.assertEquals(in_2.dataset_name, "another_ignored_input") - - self.assertEquals(new_method.outputs.count(), 1) - self.assertEquals(new_method.outputs.first().dataset_name, "empty_output") - - def test_create_with_dep(self): - """ - Test creation of a Method with dependencies. - """ - method_s = MethodSerializer(data=self.method_data_with_dep, context=self.duck_context) - self.assertTrue(method_s.is_valid()) - new_method = method_s.save() - - # Inspect the revision we just added. - self.assertEquals(new_method.revision_name, "v2") - self.assertEquals(new_method.revision_desc, "Has dependencies") - self.assertEquals(new_method.dependencies.count(), 2) - - new_dep = new_method.dependencies.get(filename="config.dat") - self.assertEquals(new_dep.requirement, self.crd) - self.assertEquals(new_dep.path, "") - - new_dep_2 = new_method.dependencies.get(filename="config_2.dat") - self.assertEquals(new_dep_2.requirement, self.crd) - self.assertEquals(new_dep_2.path, "configuration.dat") - - -@skipIfDBFeature('is_mocked') -class MethodApiTests(BaseTestCases.ApiTestCase): - fixtures = ['simple_run'] - - def setUp(self): - super(MethodApiTests, self).setUp() - - self.list_path = reverse("method-list") - self.detail_pk = 2 - self.detail_path = reverse("method-detail", - kwargs={'pk': self.detail_pk}) - self.removal_path = reverse("method-removal-plan", - kwargs={'pk': self.detail_pk}) - - self.list_view, _, _ = resolve(self.list_path) - self.detail_view, _, _ = resolve(self.detail_path) - self.removal_view, _, _ = resolve(self.removal_path) - - # noinspection PyTypeChecker - method_test_setup(self) - - def test_removal(self): - start_count = Method.objects.all().count() - - request = self.factory.delete(self.detail_path) - force_authenticate(request, user=self.kive_user) - response = self.detail_view(request, pk=self.detail_pk) - - self.assertEquals(response.status_code, status.HTTP_204_NO_CONTENT) - - end_count = Method.objects.all().count() - self.assertEquals(end_count, start_count - 1) - - def test_create(self): - request = self.factory.post(self.list_path, self.method_data, format="json") - force_authenticate(request, user=self.kive_user) - self.list_view(request) - - # Probe the resulting method. - new_method = self.dtf_mf.members.get(revision_name=self.method_data["revision_name"]) - - self.assertEquals(new_method.inputs.count(), 2) - self.assertEquals(new_method.outputs.count(), 1) - - -class MethodApiMockTests(BaseTestCases.ApiTestCase): - def setUp(self): - self.mock_viewset(MethodViewSet) - super(MethodApiMockTests, self).setUp() - - patcher = mocked_relations(MethodFamily, - CodeResourceRevision, - DockerImage) - patcher.start() - self.addCleanup(patcher.stop) - - self.list_path = reverse("method-list") - self.detail_pk = 43 - self.detail_path = reverse("method-detail", - kwargs={'pk': self.detail_pk}) - self.removal_path = reverse("method-removal-plan", - kwargs={'pk': self.detail_pk}) - - self.list_view, _, _ = resolve(self.list_path) - self.detail_view, _, _ = resolve(self.detail_path) - self.removal_view, _, _ = resolve(self.removal_path) - - Method.objects.add(Method(pk=42, - user=self.kive_kive_user, - revision_name='mA_name turnip', - revision_desc='A_desc'), - Method(pk=43, - user=self.kive_kive_user, - revision_name='mB_name', - revision_desc='B_desc'), - Method(pk=44, - user=self.kive_kive_user, - revision_name='mC_name', - revision_desc='C_desc turnip')) - - def test_list(self): - """ - Test the API list view. - """ - request = self.factory.get(self.list_path) - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - self.assertEquals(len(response.data), 3) - self.assertEquals(response.data[2]['revision_name'], 'mC_name') - - def test_detail(self): - request = self.factory.get(self.detail_path) - force_authenticate(request, user=self.kive_user) - response = self.detail_view(request, pk=self.detail_pk) - self.assertEquals(response.data['revision_name'], 'mB_name') - - def test_removal_plan(self): - request = self.factory.get(self.removal_path) - force_authenticate(request, user=self.kive_user) - response = self.removal_view(request, pk=self.detail_pk) - self.assertEquals(response.data['Methods'], 1) - - def test_filter_description(self): - """ - Test the API list view. - """ - request = self.factory.get( - self.list_path + "?filters[0][key]=description&filters[0][val]=B_desc") - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - self.assertEquals(len(response.data), 1) - self.assertEquals(response.data[0]['revision_name'], 'mB_name') - - def test_filter_smart(self): - """ - Test the API list view. - """ - request = self.factory.get( - self.list_path + "?filters[0][key]=smart&filters[0][val]=turnip") - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - self.assertEquals(len(response.data), 2) - self.assertEquals(response.data[0]['revision_name'], 'mA_name turnip') - self.assertEquals(response.data[1]['revision_desc'], 'C_desc turnip') - - def test_filter_user(self): - """ - Test the API list view. - """ - request = self.factory.get( - self.list_path + "?filters[0][key]=user&filters[0][val]=kive") - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - self.assertEquals(len(response.data), 3) - - def test_filter_unknown(self): - """ - Test the API list view. - """ - request = self.factory.get( - self.list_path + "?filters[0][key]=bogus&filters[0][val]=kive") - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - self.assertEquals({u'detail': u'Unknown filter key: bogus'}, - response.data) - - -class DockerImageApiMockTests(BaseTestCases.ApiTestCase): - def setUp(self): - self.mock_viewset(DockerImageViewSet) - super(DockerImageApiMockTests, self).setUp() - - self.list_path = reverse("dockerimage-list") - self.detail_pk = 43 - self.detail_path = reverse("dockerimage-detail", - kwargs={'pk': self.detail_pk}) - self.removal_path = reverse("dockerimage-removal-plan", - kwargs={'pk': self.detail_pk}) - - self.list_view, _, _ = resolve(self.list_path) - self.detail_view, _, _ = resolve(self.detail_path) - self.removal_view, _, _ = resolve(self.removal_path) - - DockerImage.objects.add(DockerImage(pk=42, - user=self.kive_kive_user, - name='github/alex/hello', - description='angling'), - DockerImage(pk=43, - user=self.kive_kive_user, - name='github/bob/hello', - description='boxing', - git='http://server1.com/hello.git', - tag='v1.0'), - DockerImage(pk=44, - user=self.kive_kive_user, - name='github/cindy/hello', - description='carving bob', - git='http://server2.com/hello.git', - tag='v2.0')) - - def test_list(self): - """ - Test the API list view. - """ - request = self.factory.get(self.list_path) - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - self.assertEquals(len(response.data), 3) - self.assertEquals(response.data[2]['name'], 'github/cindy/hello') - - def test_detail(self): - request = self.factory.get(self.detail_path) - force_authenticate(request, user=self.kive_user) - response = self.detail_view(request, pk=self.detail_pk) - self.assertEquals(response.data['name'], 'github/bob/hello') - - def test_removal_plan(self): - request = self.factory.get(self.removal_path) - force_authenticate(request, user=self.kive_user) - response = self.removal_view(request, pk=self.detail_pk) - self.assertEquals(response.data['DockerImages'], 1) - - def test_filter_description(self): - request = self.factory.get( - self.list_path + "?filters[0][key]=description&filters[0][val]=boxing") - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - self.assertEquals(len(response.data), 1) - self.assertEquals(response.data[0]['name'], 'github/bob/hello') - - def test_filter_smart(self): - request = self.factory.get( - self.list_path + "?filters[0][key]=smart&filters[0][val]=bob") - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - self.assertEquals(len(response.data), 2) - self.assertEquals(response.data[0]['name'], 'github/bob/hello') - self.assertEquals(response.data[1]['description'], 'carving bob') - - def test_filter_name(self): - request = self.factory.get( - self.list_path + "?filters[0][key]=name&filters[0][val]=bob") - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - self.assertEquals(len(response.data), 1) - self.assertEquals(response.data[0]['name'], 'github/bob/hello') - - def test_filter_tag(self): - request = self.factory.get( - self.list_path + "?filters[0][key]=tag&filters[0][val]=1.0") - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - self.assertEquals(len(response.data), 1) - self.assertEquals(response.data[0]['tag'], 'v1.0') - - def test_filter_git(self): - request = self.factory.get( - self.list_path + "?filters[0][key]=git&filters[0][val]=server1") - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - self.assertEquals(len(response.data), 1) - self.assertEquals(response.data[0]['git'], 'http://server1.com/hello.git') - - def test_filter_user(self): - request = self.factory.get( - self.list_path + "?filters[0][key]=user&filters[0][val]=kive") - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - self.assertEquals(len(response.data), 3) - - def test_filter_unknown(self): - request = self.factory.get( - self.list_path + "?filters[0][key]=bogus&filters[0][val]=kive") - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - self.assertEquals({u'detail': u'Unknown filter key: bogus'}, - response.data) diff --git a/kive/method/tests_mock.py b/kive/method/tests_mock.py deleted file mode 100644 index 31d3407f0..000000000 --- a/kive/method/tests_mock.py +++ /dev/null @@ -1,1041 +0,0 @@ -from django.core.files.base import ContentFile -from django.core.files.uploadedfile import SimpleUploadedFile -from mock import patch - -from django.core.exceptions import ValidationError -from django.core.urlresolvers import reverse -from django.test import TestCase - -from constants import users, groups -from django_mock_queries.query import MockSet -from django_mock_queries.mocks import mocked_relations, PatcherChain - -from container.models import Container, ContainerFamily -from kive.tests import ViewMockTestCase -from metadata.models import CompoundDatatype, KiveUser, kive_user, empty_removal_plan -from method.forms import MethodForm -from method.models import Method, MethodFamily, CodeResourceRevision, \ - CodeResource, MethodDependency, DockerImage -from transformation.models import TransformationInput, TransformationOutput,\ - XputStructure, Transformation, TransformationXput -from django.contrib.auth.models import User, Group -from pipeline.models import Pipeline, PipelineStep, PipelineFamily - -# noinspection PyUnresolvedReferences -from kive.mock_setup import convert_to_pks - - -@mocked_relations(Method, Transformation, TransformationXput, TransformationInput, TransformationOutput) -class MethodMockTests(TestCase): - def test_with_family_str(self): - """ expect "Method revision name and family name" """ - - family = MethodFamily(name="Example") - method = Method(revision_name="rounded", revision_number=3, family=family) - self.assertEqual(str(method), - "Example:3 (rounded)") - - def test_without_family_str(self): - """ - Test unicode representation when family is unset. - """ - nofamily = Method(revision_name="foo") - - self.assertEqual(str(nofamily), - "[family unset]:None (foo)") - - def test_display_name(self): - method = Method(revision_number=1, revision_name='Example') - - self.assertEqual(method.display_name, '1: Example') - - def test_display_name_without_revision_name(self): - method = Method(revision_number=1) - - self.assertEqual(method.display_name, '1: ') - - def test_no_inputs_checkInputIndices_good(self): - """ - Method with no inputs defined should have - check_input_indices() return with no exception. - """ - - driver = CodeResourceRevision(coderesource=CodeResource()) - - foo = Method(driver=driver, family=MethodFamily()) - - # check_input_indices() should not raise a ValidationError - foo.check_input_indices() - foo.clean() - - def test_single_valid_input_checkInputIndices_good(self): - """ - Method with a single, 1-indexed input should have - check_input_indices() return with no exception. - """ - - driver = CodeResourceRevision(coderesource=CodeResource()) - - foo = Method(driver=driver, family=MethodFamily()) - inp = foo.inputs.create(dataset_idx=1) - inp.transformationinput = inp - - # check_input_indices() should not raise a ValidationError - foo.check_input_indices() - foo.clean() - - def test_many_ordered_valid_inputs_checkInputIndices_good(self): - """ - Test check_input_indices on a method with several inputs, - correctly indexed and in order. - """ - - driver = CodeResourceRevision(coderesource=CodeResource()) - - foo = Method(driver=driver, family=MethodFamily()) - for i in range(3): - inp = foo.inputs.create(dataset_idx=i + 1) - inp.transformationinput = inp - - # check_input_indices() should not raise a ValidationError - foo.check_input_indices() - foo.clean() - - def test_many_valid_inputs_scrambled_checkInputIndices_good(self): - """ - Test check_input_indices on a method with several inputs, - correctly indexed and in scrambled order. - """ - - driver = CodeResourceRevision(coderesource=CodeResource()) - - foo = Method(driver=driver, family=MethodFamily()) - for i in (3, 1, 2): - inp = foo.inputs.create(dataset_idx=i) - inp.transformationinput = inp - - # check_input_indices() should not raise a ValidationError - foo.check_input_indices() - foo.clean() - - def test_one_invalid_input_checkInputIndices_bad(self): - """ - Test input index check, one badly-indexed input case. - """ - - driver = CodeResourceRevision(coderesource=CodeResource()) - - foo = Method(driver=driver, family=MethodFamily()) - inp = foo.inputs.create(dataset_idx=4) - inp.transformationinput = inp - - # check_input_indices() should raise a ValidationError - self.assertRaisesRegexp( - ValidationError, - "Inputs are not consecutively numbered starting from 1", - foo.check_input_indices) - - self.assertRaisesRegexp( - ValidationError, - "Inputs are not consecutively numbered starting from 1", - foo.clean) - - def test_many_nonconsective_inputs_scrambled_checkInputIndices_bad(self): - """Test input index check, badly-indexed multi-input case.""" - - driver = CodeResourceRevision(coderesource=CodeResource()) - - foo = Method(driver=driver, family=MethodFamily()) - for i in (2, 6, 1): - inp = foo.inputs.create(dataset_idx=i) - inp.transformationinput = inp - - self.assertRaisesRegexp( - ValidationError, - "Inputs are not consecutively numbered starting from 1", - foo.check_input_indices) - - self.assertRaisesRegexp( - ValidationError, - "Inputs are not consecutively numbered starting from 1", - foo.clean) - - def test_no_outputs_checkOutputIndices_good(self): - """Test output index check, one well-indexed output case.""" - driver = CodeResourceRevision(coderesource=CodeResource()) - - foo = Method(driver=driver, family=MethodFamily()) - - foo.check_output_indices() - foo.clean() - - def test_one_valid_output_checkOutputIndices_good(self): - """Test output index check, one well-indexed output case.""" - - driver = CodeResourceRevision(coderesource=CodeResource()) - - foo = Method(driver=driver, family=MethodFamily()) - out = foo.outputs.create(dataset_idx=1) - out.transformationoutput = out - - foo.check_output_indices() - foo.clean() - - def test_many_valid_outputs_scrambled_checkOutputIndices_good(self): - """Test output index check, well-indexed multi-output (scrambled order) case.""" - - driver = CodeResourceRevision(coderesource=CodeResource()) - - foo = Method(driver=driver, family=MethodFamily()) - for i in (3, 1, 2): - out = foo.outputs.create(dataset_idx=i) - out.transformationoutput = out - - foo.check_output_indices() - foo.clean() - - def test_one_invalid_output_checkOutputIndices_bad(self): - """Test output index check, one badly-indexed output case.""" - - driver = CodeResourceRevision(coderesource=CodeResource()) - - foo = Method(driver=driver, family=MethodFamily()) - out = foo.outputs.create(dataset_idx=4) - out.transformationoutput = out - - self.assertRaisesRegexp( - ValidationError, - "Outputs are not consecutively numbered starting from 1", - foo.check_output_indices) - - self.assertRaisesRegexp( - ValidationError, - "Outputs are not consecutively numbered starting from 1", - foo.clean) - - def test_many_invalid_outputs_scrambled_checkOutputIndices_bad(self): - """Test output index check, badly-indexed multi-output case.""" - - driver = CodeResourceRevision(coderesource=CodeResource()) - - foo = Method(driver=driver, family=MethodFamily()) - for i in (2, 6, 1): - out = foo.outputs.create(dataset_idx=i) - out.transformationoutput = out - - self.assertRaisesRegexp( - ValidationError, - "Outputs are not consecutively numbered starting from 1", - foo.check_output_indices) - - self.assertRaisesRegexp( - ValidationError, - "Outputs are not consecutively numbered starting from 1", - foo.clean) - - def create_parent(self): - parent = Method() - parent.inputs = MockSet(name='parent.inputs', - model=TransformationInput) - parent.outputs = MockSet(name='parent.outputs', - model=TransformationOutput) - for i in range(2): - inp = parent.inputs.create(dataset_idx=i + 1) - inp.transformationinput = inp - for i in range(3): - out = parent.outputs.create(dataset_idx=i + 1) - out.transformationoutput = out - return parent - - def test_copy_io_from_parent(self): - parent = self.create_parent() - expected_inputs = {inp.dataset_idx for inp in parent.inputs} - expected_outputs = {out.dataset_idx for out in parent.outputs} - - foo = Method(revision_parent=parent) - foo.copy_io_from_parent() - - self.assertEqual(expected_inputs, - {inp.dataset_idx for inp in foo.inputs}) - self.assertEqual(expected_outputs, - {out.dataset_idx for out in foo.outputs}) - - def test_copy_io_from_no_parent(self): - foo = Method() - foo.copy_io_from_parent() - - self.assertEqual(set(), - {inp.dataset_idx for inp in foo.inputs}) - self.assertEqual(set(), - {out.dataset_idx for out in foo.outputs}) - - def test_copy_io_from_parent_does_not_replace_inputs(self): - parent = self.create_parent() - - foo = Method(revision_parent=parent) - foo.inputs.create(dataset_idx=1) - - expected_inputs = {inp.dataset_idx for inp in foo.inputs} - expected_outputs = {out.dataset_idx for out in foo.outputs} - - foo.copy_io_from_parent() - - self.assertEqual(expected_inputs, - {inp.dataset_idx for inp in foo.inputs}) - self.assertEqual(expected_outputs, - {out.dataset_idx for out in foo.outputs}) - - def test_copy_io_from_parent_does_not_replace_outputs(self): - parent = self.create_parent() - - foo = Method(revision_parent=parent) - foo.outputs.create(dataset_idx=1) - - expected_inputs = {inp.dataset_idx for inp in foo.inputs} - expected_outputs = {out.dataset_idx for out in foo.outputs} - - foo.copy_io_from_parent() - - self.assertEqual(expected_inputs, - {inp.dataset_idx for inp in foo.inputs}) - self.assertEqual(expected_outputs, - {out.dataset_idx for out in foo.outputs}) - - @mocked_relations(XputStructure, CompoundDatatype) - def test_copy_io_from_parent_with_structure(self): - cdt = CompoundDatatype() - min_row = 1 - max_row = 100 - structure = XputStructure(compounddatatype=cdt, - min_row=min_row, - max_row=max_row) - parent = self.create_parent() - - def get_structure(xput_self): - if xput_self.dataset_idx == 1: - return structure - # noinspection PyUnresolvedReferences - raise XputStructure.DoesNotExist - - TransformationXput.structure = property(get_structure) - expected_inputs = {inp.dataset_idx for inp in parent.inputs} - expected_outputs = {out.dataset_idx for out in parent.outputs} - - foo = Method(revision_parent=parent) - foo.copy_io_from_parent() - - self.assertEqual(expected_inputs, - {inp.dataset_idx for inp in foo.inputs}) - self.assertEqual(expected_outputs, - {out.dataset_idx for out in foo.outputs}) - # noinspection PyUnresolvedReferences - create_args = XputStructure.objects.create.call_args_list # @UndefinedVariable - self.assertEqual(2, len(create_args)) - _args, kwargs = create_args[0] - self.assertEqual(100, kwargs['max_row']) - - def test_identical_self(self): - """A Method should be identical to itself.""" - m = Method(driver=CodeResourceRevision(), user=User()) - self.assertTrue(m.is_identical(m)) - - def test_identical(self): - driver = CodeResourceRevision() - user = User() - m1 = Method(revision_name='A', driver=driver, user=user) - for i in range(2): - inp = m1.inputs.create(dataset_name='a_in_{}'.format(i), - dataset_idx=i + 1) - inp.transformationinput = inp - for i in range(3): - out = m1.outputs.create(dataset_name='a_out_{}'.format(i), - dataset_idx=i + 1) - out.transformationoutput = out - - m2 = Method(revision_name='B', driver=driver, user=user) - for i in range(2): - inp = m2.inputs.create(dataset_name='b_in_{}'.format(i), - dataset_idx=i + 1) - inp.transformationinput = inp - for i in range(3): - out = m2.outputs.create(dataset_name='b_in_{}'.format(i), - dataset_idx=i + 1) - out.transformationoutput = out - - self.assertTrue(m1.is_identical(m2)) - - def test_identical_when_drivers_unmatched(self): - driver1 = CodeResourceRevision() - driver2 = CodeResourceRevision() - user = User() - m1 = Method(revision_name='A', driver=driver1, user=user) - for i in range(2): - inp = m1.inputs.create(dataset_name='a_in_{}'.format(i), - dataset_idx=i + 1) - inp.transformationinput = inp - for i in range(3): - out = m1.outputs.create(dataset_name='a_out_{}'.format(i), - dataset_idx=i + 1) - out.transformationoutput = out - - m2 = Method(revision_name='B', driver=driver2, user=user) - for i in range(2): - inp = m2.inputs.create(dataset_name='b_in_{}'.format(i), - dataset_idx=i + 1) - inp.transformationinput = inp - for i in range(3): - out = m2.outputs.create(dataset_name='b_in_{}'.format(i), - dataset_idx=i + 1) - out.transformationoutput = out - - self.assertFalse(m1.is_identical(m2)) - - def test_identical_when_inputs_unmatched(self): - driver = CodeResourceRevision() - user = User() - m1 = Method(revision_name='A', driver=driver, user=user) - for i in range(1): - inp = m1.inputs.create(dataset_name='a_in_{}'.format(i), - dataset_idx=i + 1) - inp.transformationinput = inp - for i in range(3): - out = m1.outputs.create(dataset_name='a_out_{}'.format(i), - dataset_idx=i + 1) - out.transformationoutput = out - - m2 = Method(revision_name='B', driver=driver, user=user) - for i in range(2): - inp = m2.inputs.create(dataset_name='b_in_{}'.format(i), - dataset_idx=i + 1) - inp.transformationinput = inp - for i in range(3): - out = m2.outputs.create(dataset_name='b_in_{}'.format(i), - dataset_idx=i + 1) - out.transformationoutput = out - - self.assertFalse(m1.is_identical(m2)) - - def test_identical_when_outputs_unmatched(self): - driver = CodeResourceRevision() - user = User() - m1 = Method(revision_name='A', driver=driver, user=user) - for i in range(2): - inp = m1.inputs.create(dataset_name='a_in_{}'.format(i), - dataset_idx=i + 1) - inp.transformationinput = inp - for i in range(2): - out = m1.outputs.create(dataset_name='a_out_{}'.format(i), - dataset_idx=i + 1) - out.transformationoutput = out - - m2 = Method(revision_name='B', driver=driver, user=user) - for i in range(2): - inp = m2.inputs.create(dataset_name='b_in_{}'.format(i), - dataset_idx=i + 1) - inp.transformationinput = inp - for i in range(3): - out = m2.outputs.create(dataset_name='b_in_{}'.format(i), - dataset_idx=i + 1) - out.transformationoutput = out - - self.assertFalse(m1.is_identical(m2)) - - -class MethodDependencyMockTests(TestCase): - def setUp(self): - patcher = mocked_relations(Method, MethodDependency, Transformation) - patcher.start() - self.addCleanup(patcher.stop) - driver = CodeResourceRevision( - coderesource=CodeResource(filename='driver.py')) - self.method = Method(driver=driver, family=MethodFamily()) - self.dependency = self.add_dependency('helper.py') - - def add_dependency(self, filename): - helper = CodeResourceRevision( - coderesource=CodeResource(filename=filename)) - dependency = self.method.dependencies.create(requirement=helper) - dependency.method = self.method - return dependency - - def test_dependency_depends_on_nothing_clean_good(self): - self.method.dependencies.clear() - - self.method.clean() - - def test_dependency_current_folder_same_name_clean_bad(self): - """ - A depends on B - current folder, same name - """ - # We're giving the dependency a conflicting filename. - self.dependency.filename = self.method.driver.coderesource.filename - - self.assertRaisesRegexp(ValidationError, - "Conflicting dependencies", - self.method.clean) - - def test_dependency_current_folder_different_name_clean_good(self): - """ - 1 depends on 2 - current folder, different name - """ - self.dependency.filename = 'different_name.py' - - self.method.clean() - - def test_dependency_inner_folder_same_name_clean_good(self): - """ - 1 depends on 2 - different folder, same name - """ - self.dependency.path = 'subfolder' - self.dependency.filename = self.method.driver.coderesource.filename - - self.method.clean() - - def test_dependency_inner_folder_different_name_clean_good(self): - """ - 1 depends on 2 - different folder, different name - """ - self.dependency.path = 'subfolder' - self.dependency.filename = 'different_name.py' - - self.method.clean() - - def test_dependency_A_depends_BC_same_folder_no_conflicts_clean_good(self): - """ - A depends on B, A depends on C - BC in same folder as A - Nothing conflicts - """ - self.add_dependency('helper2.py') - - self.method.clean() - - def test_dependency_A_depends_BC_same_folder_B_conflicts_with_C_clean_bad(self): - """ - A depends on B, A depends on C - BC in same folder as A, BC conflict - """ - self.dependency.filename = 'same_name.py' - self.add_dependency(self.dependency.filename) - - self.assertRaisesRegexp( - ValidationError, - "Conflicting dependencies", - self.method.clean) - - def test_list_all_filepaths_unnested_dep_blank_filename(self): - """ - List all filepaths when dependency has no filename set and is not in a subdirectory. - """ - expected_filepaths = ['driver.py', 'helper.py'] - - filepaths = self.method.list_all_filepaths() - - self.assertEqual(expected_filepaths, filepaths) - - def test_list_all_filepaths_nested_dep_blank_filename(self): - """ - List all filepaths when dependency has no filename set and is in a subdirectory. - """ - self.dependency.path = 'nest_folder' - expected_filepaths = ['driver.py', 'nest_folder/helper.py'] - - filepaths = self.method.list_all_filepaths() - - self.assertEqual(expected_filepaths, filepaths) - - def test_list_all_filepaths_unnested_dep_specified_filename(self): - """List all filepaths when dependency has a custom filename and is not in a subdirectory. - """ - self.dependency.filename = 'foo.py' - expected_filepaths = ['driver.py', 'foo.py'] - - filepaths = self.method.list_all_filepaths() - - self.assertEqual(expected_filepaths, filepaths) - - def test_list_all_filepaths_nested_dep_specified_filename(self): - """ - List all filepaths when dependency has a custom filename and is in a subdirectory. - """ - self.dependency.path = 'nest_folder' - self.dependency.filename = 'foo.py' - expected_filepaths = ['driver.py', 'nest_folder/foo.py'] - - filepaths = self.method.list_all_filepaths() - - self.assertEqual(expected_filepaths, filepaths) - - -class MethodUpdateMockTests(TestCase): - def setUp(self): - patcher = mocked_relations(Method, MethodFamily, Transformation) - patcher.start() - self.addCleanup(patcher.stop) - self.family = MethodFamily() - self.old_method = self.family.members.create(family=self.family, - revision_number=1, - id=101) - self.old_method.method = self.old_method - - self.new_method = self.family.members.create(family=self.family, - revision_number=2, - id=102) - self.new_method.method = self.new_method - - def test_find_update_not_found(self): - update = self.new_method.find_update() - - self.assertEqual(None, update) - - def test_find_update(self): - update = self.old_method.find_update() - - self.assertEqual(self.new_method, update) - - @mocked_relations(Pipeline) - def test_find_update_not_found_from_transformation(self): - transformation = Transformation(id=self.new_method.id) - transformation.method = self.new_method - update = transformation.find_update() - - self.assertEqual(update, None) - - -class CodeResourceViewMockTests(ViewMockTestCase): - def setUp(self): - super(CodeResourceViewMockTests, self).setUp() - patcher = mocked_relations(KiveUser, - CodeResource, - CodeResourceRevision, - User, - Group) - patcher.start() - self.addCleanup(patcher.stop) - - # noinspection PyUnresolvedReferences - patchers = [patch.object(CodeResource._meta, - 'default_manager', - CodeResource.objects), - patch.object(CodeResourceRevision._meta, - 'default_manager', - CodeResource.objects)] - - def dummy_save(r): - r.id = id(r) - - # noinspection PyUnresolvedReferences - patchers.append(patch.object(CodeResource, 'save', dummy_save)) - patcher = PatcherChain(patchers) - patcher.start() - self.addCleanup(patcher.stop) - - self.client = self.create_client() - self.dev_group = Group(pk=groups.DEVELOPERS_PK) - self.everyone = Group(pk=groups.EVERYONE_PK) - Group.objects.add(self.dev_group, self.everyone) - self.user = kive_user() - self.user.groups.add(self.dev_group) - self.content_file = ContentFile('some text', 'existing.txt') - self.code_resource = CodeResource(pk='99', - user=self.user, - name='existing', - filename='existing.txt') - self.code_resource._state.adding = False - - self.other_user = User(pk=5) - self.other_code_resource = CodeResource(pk='150', user=self.other_user) - CodeResource.objects.add(self.code_resource, self.other_code_resource) - - self.code_resource_revision = CodeResourceRevision( - pk='199', - user=self.user, - content_file=self.content_file) - self.code_resource_revision.coderesource = self.code_resource - self.other_code_resource_revision = CodeResourceRevision( - pk='200', - user=self.other_user) - self.other_code_resource_revision.coderesource = self.other_code_resource - self.other_code_resource.revisions.add(self.other_code_resource_revision) - CodeResourceRevision.objects.add(self.code_resource_revision, - self.other_code_resource_revision) - k = KiveUser(pk=users.KIVE_USER_PK) - k.groups.add(self.dev_group) - KiveUser.objects.add(k) - - def test_resources(self): - response = self.client.get(reverse('resources')) - - self.assertEqual(200, response.status_code) - self.assertFalse(response.context['is_user_admin']) - - def test_resources_admin(self): - self.user.is_staff = True - - response = self.client.get(reverse('resources')) - - self.assertEqual(200, response.status_code) - self.assertTrue(response.context['is_user_admin']) - - def test_resource_revisions_404(self): - response = self.client.get(reverse('resource_revisions', - kwargs=dict(pk='1000'))) - - self.assertEqual(404, response.status_code) - - def test_resource_revisions(self): - response = self.client.get(reverse('resource_revisions', - kwargs=dict(pk='99'))) - - self.assertEqual(200, response.status_code) - self.assertEqual(self.code_resource, response.context['coderesource']) - - def test_resource_revisions_not_accessible(self): - other_id = self.other_code_resource.pk - response = self.client.get(reverse('resource_revisions', - kwargs=dict(pk=other_id))) - - self.assertEqual(404, response.status_code) - - def test_resource_revisions_accessible(self): - self.other_code_resource.groups_allowed.add(self.dev_group) - other_id = self.other_code_resource.pk - response = self.client.get(reverse('resource_revisions', - kwargs=dict(pk=other_id))) - - self.assertEqual(200, response.status_code) - self.assertNotIn('revisions', response.context) - - def test_resource_revisions_accessible_with_child(self): - self.other_code_resource.groups_allowed.add(self.dev_group) - self.other_code_resource_revision.groups_allowed.add(self.dev_group) - other_id = self.other_code_resource.pk - response = self.client.get(reverse('resource_revisions', - kwargs=dict(pk=other_id))) - - self.assertEqual(200, response.status_code) - self.assertIn('revisions', response.context) - - def test_resource_add(self): - response = self.client.get(reverse('resource_add')) - - self.assertEqual(200, response.status_code) - self.assertIn('resource_form', response.context) - - def test_resource_add_post(self): - filename = "added.txt" - upload_file = SimpleUploadedFile(filename, "Hello, World!".encode(encoding="utf-8")) - response = self.client.post( - reverse('resource_add'), - data=dict(resource_name='hello.txt', - content_file=upload_file)) - - self.assertEqual(302, response.status_code) - self.assertEqual('/resources', response.url) - - def test_resource_revision_add(self): - response = self.client.get(reverse('resource_revision_add', - kwargs=dict(pk='199'))) - - self.assertEqual(200, response.status_code) - self.assertIn('revision_form', response.context) - - def test_resource_revision_add_post(self): - filename = "added1.txt" - upload_file = SimpleUploadedFile(filename, "Hello, World!".encode(encoding="utf-8")) - response = self.client.post( - reverse('resource_revision_add', kwargs=dict(pk='199')), - data=dict(content_file=upload_file)) - - self.assertEqual(302, response.status_code) - self.assertEqual('/resources', response.url) - - def test_resource_revision_view(self): - response = self.client.get(reverse('resource_revision_view', - kwargs=dict(pk='199'))) - - self.assertEqual(200, response.status_code) - self.assertEqual(self.code_resource_revision, response.context['revision']) - - -class MethodViewMockTests(ViewMockTestCase): - def setUp(self): - super(MethodViewMockTests, self).setUp() - patcher = mocked_relations(KiveUser, - MethodFamily, - Method, - CodeResource, - CodeResourceRevision, - CompoundDatatype, - ContainerFamily, - Container, - Transformation, - TransformationInput, - TransformationOutput, - User, - Group) - patcher.start() - self.addCleanup(patcher.stop) - - # noinspection PyUnresolvedReferences - patcher = patch.object(MethodFamily._meta, - 'default_manager', - MethodFamily.objects) - patcher.start() - self.addCleanup(patcher.stop) - - self.client = self.create_client() - self.dev_group = Group(pk=groups.DEVELOPERS_PK) - self.everyone = Group(pk=groups.EVERYONE_PK) - Group.objects.add(self.dev_group, self.everyone) - self.user = kive_user() - self.user.groups.add(self.dev_group) - self.other_user = User(pk=5) - - self.method_family = MethodFamily(pk='99', - user=self.user) - MethodFamily.objects.add(self.method_family) - - self.driver = CodeResourceRevision(user=self.user) - self.driver.pk = 1337 # needed for viewing a method - self.driver.coderesource = CodeResource() - self.method = Method(pk='199', user=self.user) - self.method.driver = self.driver - self.method.family = self.method_family - Method.objects.add(self.method) - KiveUser.objects.add(KiveUser(pk=users.KIVE_USER_PK)) - - def test_method_families(self): - response = self.client.get(reverse('method_families')) - - self.assertEqual(200, response.status_code) - self.assertFalse(response.context['is_user_admin']) - - def test_method_families_admin(self): - self.user.is_staff = True - - response = self.client.get(reverse('method_families')) - - self.assertEqual(200, response.status_code) - self.assertTrue(response.context['is_user_admin']) - - def test_methods_404(self): - response = self.client.get(reverse('methods', - kwargs=dict(pk='1000'))) - - self.assertEqual(404, response.status_code) - - def test_methods(self): - response = self.client.get(reverse('methods', - kwargs=dict(pk='99'))) - - self.assertEqual(200, response.status_code) - self.assertEqual(self.method_family, response.context['family']) - - def test_method_view(self): - response = self.client.get(reverse('method_view', - kwargs=dict(pk='199'))) - - self.assertEqual(200, response.status_code) - self.assertEqual(self.method, response.context['method']) - - def test_method_new(self): - response = self.client.get(reverse('method_new')) - - self.assertEqual(200, response.status_code) - self.assertEqual(None, response.context['family']) - - # noinspection PyUnresolvedReferences - @patch.object(TransformationXput, 'clean', lambda xput: None) - def test_method_new_post(self): - response = self.client.post( - reverse('method_new'), - data=dict(name='foo', - memory='100', - threads='1', - container='20', - reusable='1', - dataset_name_in_0='in_csv', - compounddatatype_in_0='__raw__', - min_row_in_0='', - max_row_in_0='', - dataset_name_out_0='out_csv', - compounddatatype_out_0='__raw__', - min_row_out_0='', - max_row_out_0='')) - - expected_status = 302 - if response.status_code != expected_status: - for key in response.context.keys(): - if key.endswith('_form'): - form = response.context[key] - self.assertEqual({}, form.errors) - self.assertEqual(expected_status, response.status_code) - self.assertEqual('/methods/None', response.url) - - def test_method_revise(self): - response = self.client.get(reverse('method_revise', - kwargs=dict(pk='199'))) - - self.assertEqual(200, response.status_code) - self.assertEqual(self.method_family, response.context['family']) - - def test_method_revise_access_denied(self): - """ Hides ungranted code revisions. """ - revision1 = CodeResourceRevision(pk=101, - revision_name='alpha', - revision_number=1, - user=self.user) - revision2 = CodeResourceRevision(pk=102, - revision_name='beta', - revision_number=2, - user=self.other_user) - self.driver.coderesource.revisions.add(revision1, revision2) - - response = self.client.get(reverse('method_revise', - kwargs=dict(pk='199'))) - - self.assertEqual(200, response.status_code) - revisions = response.context['method_form']['driver_revisions'] - self.assertEqual([('101', '1: alpha')], - revisions.field.widget.choices) - - def test_method_revise_access_granted(self): - """ Shows granted code revisions. """ - revision1 = CodeResourceRevision(pk=101, - revision_name='alpha', - revision_number=1, - user=self.user) - revision2 = CodeResourceRevision(pk=102, - revision_name='beta', - revision_number=2, - user=self.other_user) - revision2.users_allowed.add(self.user) - self.driver.coderesource.revisions.add(revision1, revision2) - - response = self.client.get(reverse('method_revise', - kwargs=dict(pk='199'))) - - self.assertEqual(200, response.status_code) - revisions = response.context['method_form']['driver_revisions'] - self.assertEqual([('101', '1: alpha'), - ('102', '2: beta')], - revisions.field.widget.choices) - - -class DockerImageViewMockTests(ViewMockTestCase): - def setUp(self): - super(DockerImageViewMockTests, self).setUp() - patcher = mocked_relations(KiveUser, - DockerImage, - User, - Group) - patcher.start() - self.addCleanup(patcher.stop) - - self.client = self.create_client() - self.dev_group = Group(pk=groups.DEVELOPERS_PK) - self.everyone = Group(pk=groups.EVERYONE_PK) - Group.objects.add(self.dev_group, self.everyone) - self.user = kive_user() - self.user.groups.add(self.dev_group) - self.other_user = User(pk=5) - - self.docker_image = DockerImage(pk='99', - name='git/joe/hello', - tag='v1', - git='http://server1.com/joe/hello.git', - user=self.user) - DockerImage.objects.add(self.docker_image) - - KiveUser.objects.add(KiveUser(pk=users.KIVE_USER_PK)) - - def test_docker_images(self): - response = self.client.get(reverse('docker_images')) - - self.assertEqual(200, response.status_code) - self.assertFalse(response.context['is_user_admin']) - - def test_docker_images_admin(self): - self.user.is_staff = True - - response = self.client.get(reverse('docker_images')) - - self.assertEqual(200, response.status_code) - self.assertTrue(response.context['is_user_admin']) - - def test_docker_image_view(self): - response = self.client.get(reverse('docker_image_view', - kwargs=dict(image_id='99'))) - - self.assertEqual(200, response.status_code) - self.assertEqual(self.docker_image, response.context['docker_image']) - - def test_docker_image_update(self): - new_description = 'new description' - response = self.client.post(reverse('docker_image_view', - kwargs=dict(image_id='99')), - data=dict(description=new_description)) - - self.assertEqual(302, response.status_code) - self.assertEqual(new_description, self.docker_image.description) - - # noinspection PyUnusedLocal - @patch('method.models.check_output') - def test_docker_image_add(self, mock_check_output): - new_description = 'new description' - expected_name = 'git/alex/howdy' - response = self.client.post( - reverse('docker_image_add'), - data=dict(name=expected_name, - tag='v9', - git='http://server1.com/alex/howdy.git', - description=new_description)) - - self.assertEqual(302, response.status_code) - - def test_docker_image_update_too_long(self): - new_description = 'X' * 2001 - response = self.client.post(reverse('docker_image_view', - kwargs=dict(image_id='99')), - data=dict(description=new_description)) - - self.assertEqual(200, response.status_code) - self.assertEqual( - {'description': ['Ensure this value has at most 2000 characters' - ' (it has 2001).']}, - response.context['docker_image_form'].errors) - - def test_docker_image_new(self): - response = self.client.get(reverse('docker_image_add')) - - self.assertEqual(200, response.status_code) - - -@mocked_relations(DockerImage, Method, PipelineStep, Pipeline) -class DockerImageMockTests(TestCase): - def test_build_removal_plan_for_unused_image(self): - image = DockerImage.objects.create(id=99, name='doomed') - DockerImage.objects.create(id=100, name='untouched') - expected_plan = empty_removal_plan() - expected_plan['DockerImages'].add(image) - - plan = image.build_removal_plan() - - self.assertEqual(expected_plan, plan) - - def test_build_removal_plan_for_used_image(self): - image = DockerImage(id=99, name='doomed') - method = image.methods.create(transformation_ptr_id=100) - step = method.pipelinesteps.create(id=101) - step.pipeline = Pipeline(transformation_ptr_id=102) - step.pipeline.family = PipelineFamily() - - expected_plan = empty_removal_plan() - expected_plan['DockerImages'].add(image) - expected_plan['Methods'].add(method) - expected_plan['Pipelines'].add(step.pipeline) - - plan = image.build_removal_plan() - - self.assertEqual(expected_plan, plan) diff --git a/kive/pipeline/tests.py b/kive/pipeline/tests.py deleted file mode 100644 index 910860b40..000000000 --- a/kive/pipeline/tests.py +++ /dev/null @@ -1,3067 +0,0 @@ -""" -Shipyard unit tests pertaining to Pipeline and its relatives. -""" - -import os.path -import re -import shutil -import tempfile - -from django.contrib.auth.models import User, Group -from django.core.exceptions import ValidationError -from django.core.files.base import File -from django.core.urlresolvers import resolve -from django.test import TestCase, skipIfDBFeature -from django.utils import timezone -from rest_framework import status -from rest_framework.reverse import reverse -from rest_framework.test import force_authenticate - -from archive.models import ExecLog -from constants import datatypes -from kive.tests import BaseTestCases, DuckContext -from metadata.models import CompoundDatatype, CompoundDatatypeMember, \ - Datatype, kive_user, everyone_group -from method.models import Method, MethodFamily, MethodDependency, \ - CodeResource, CodeResourceRevision -from pipeline.models import Pipeline, PipelineFamily, \ - PipelineStep, PipelineStepInputCable, \ - PipelineOutputCable -from pipeline.serializers import PipelineSerializer,\ - PipelineStepUpdateSerializer, PipelineFamilySerializer -import kive.testing_utils as tools - -samplecode_path = tools.samplecode_path - - -@skipIfDBFeature('is_mocked') -class PipelineTestCase(TestCase): - """ - Set up a database state for unit testing Pipeline. - """ - def setUp(self): - """Set up default database state for Pipeline unit testing.""" - tools.create_pipeline_test_environment(self) - - def tearDown(self): - tools.destroy_pipeline_test_environment(self) - - -class PipelineFamilyTests(PipelineTestCase): - - def test_unicode(self): - """ - unicode() for PipelineFamily should display it's name - """ - self.assertEqual(unicode(self.DNAcomp_pf), "DNAcomplement") - - def test_delete_pipeline_family(self): - """Can I delete a PipelineFamily?""" - PipelineFamily.objects.first().delete() - - # def test_published_version_display_name_is_none(self): - # family = PipelineFamily.objects.get(name='DNAcomplement') - # self.assertIsNone(family.published_version_display_name) - - # def test_published_version_display_name(self): - # family = PipelineFamily.objects.get(name='DNAcomplement') - # family.published_version = family.members.last() #oldest version - # family.clean() - # family.save() - # - # reloaded = PipelineFamily.objects.get(pk=family.pk) - # - # self.assertEqual("1: v1", reloaded.published_version_display_name) - - -class PipelineTests(PipelineTestCase): - """Tests for basic Pipeline functionality.""" - def test_delete_pipeline(self): - """Deleting a Pipeline is possible.""" - family = PipelineFamily(user=self.user) - family.save() - pipeline = Pipeline(family=family, user=self.user) - pipeline.save() - pipeline.delete() - - -class PipelineStepTests(PipelineTestCase): - - def test_pipelineStep_without_pipeline_set_unicode(self): - """Test unicode representation when no pipeline is set.""" - nopipeline = PipelineStep(step_num=2) - self.assertEquals(unicode(nopipeline), "2: ") - - def test_pipelineStep_with_pipeline_set_unicode(self): - """Test unicode representation when pipeline is set.""" - pipelineset = self.DNAcompv1_p.steps.get(step_num=1) - self.assertEquals(unicode(pipelineset), "1: ") - - def test_pipelineStep_invalid_request_for_future_step_data_clean(self): - """Bad cabling: step requests data from after its execution step.""" - foo = Pipeline(family=self.DNAcomp_pf, revision_name="foo", revision_desc="Foo version", user=self.user) - foo.save() - foo.create_input(compounddatatype=self.DNAinput_cdt, - dataset_name="oneinput", - dataset_idx=1) - - # Step 1 invalidly requests data from step 2 - step1 = foo.steps.create(transformation=self.DNAcompv2_m, step_num=1) - cable = step1.cables_in.create( - dest=step1.transformation.inputs.get(dataset_name="input"), - source_step=2, - source=foo.inputs.get(dataset_name="oneinput")) - - self.assertRaisesRegexp( - ValidationError, - "Step 1 requests input from a later step", - cable.clean) - self.assertRaisesRegexp( - ValidationError, - "Step 1 requests input from a later step", - step1.clean) - - def test_pipelineStep_oneStep_cable_to_invalid_step_input_clean(self): - """Bad cabling: step cables to input not belonging to its transformation.""" - - # Define Pipeline - foo = Pipeline(family=self.DNAcomp_pf, revision_name="foo", revision_desc="Foo version", user=self.user) - foo.save() - - # Define Pipeline input - foo.create_input(compounddatatype=self.DNAinput_cdt, - dataset_name="oneinput", - dataset_idx=1) - - # Create a step composed of method DNAcompv2_m - step1 = foo.steps.create(transformation=self.DNAcompv2_m, step_num=1) - - # Reference an invalid input name from step 0 - cable = step1.cables_in.create( - dest=self.script_1_method.inputs.get(dataset_name="input_tuple"), - source_step=0, - source=foo.inputs.get(dataset_name="oneinput")) - - self.assertRaisesRegexp( - ValidationError, - 'Transformation at step 1 does not have input ".*"', - cable.clean) - self.assertRaisesRegexp( - ValidationError, - 'Transformation at step 1 does not have input ".*"', - step1.clean) - - def test_pipelineStep_oneStep_valid_cabling_with_valid_delete_clean(self): - """Test good step cabling with deleted dataset, one-step pipeline.""" - - # Define pipeline - foo = Pipeline(family=self.DNAcomp_pf, revision_name="foo", revision_desc="Foo version", user=self.user) - foo.save() - - # Define Pipeline input "oneinput" - foo.create_input(compounddatatype=self.DNAinput_cdt, - dataset_name="oneinput", - dataset_idx=1) - - # Add a step - step1 = foo.steps.create(transformation=self.DNAcompv2_m, step_num=1) - - # Map Pipeline input to step 1 - step1.cables_in.create( - dest=step1.transformation.inputs.get(dataset_name="input"), - source_step=0, - source=foo.inputs.get(dataset_name="oneinput")) - - # Mark step 1 "output" as deletable - # step 1 "output" is defined by DNAcompv2_m - step1.add_deletion( - step1.transformation.outputs.get(dataset_name="output")) - - self.assertEquals(step1.clean(), None) - - def test_pipelineStep_oneStep_valid_cabling_bad_delete_clean(self): - """Bad cabling: deleting dataset that doesn't belong to this step, one-step pipeline.""" - - # Define pipeline - foo = Pipeline(family=self.DNAcomp_pf, revision_name="foo", revision_desc="Foo version", user=self.user) - foo.save() - - # Add a valid pipeline input - foo.create_input(compounddatatype=self.DNAinput_cdt, dataset_name="oneinput", dataset_idx=1) - - # Define valid pipeline step - step1 = foo.steps.create(transformation=self.DNAcompv2_m, step_num=1) - - # Create input cabling for this step - step1.cables_in.create( - dest=step1.transformation.inputs.get(dataset_name="input"), - source_step=0, - source=foo.inputs.get(dataset_name="oneinput")) - - # Reference TransformationOutput not belonging to this step's - # transformation. - step1.add_deletion(self.script_2_method.outputs.all()[0]) - self.assertRaisesRegexp( - ValidationError, - 'Transformation at step 1 does not have output ".*"', - step1.clean) - - def test_pipelineStep_oneStep_cabling_directly_self_referential_transformation_clean(self): - """Bad step: pipeline step contains the parent pipeline directly.""" - - # Define pipeline - foo = Pipeline(family=self.DNAcomp_pf, revision_name="foo", revision_desc="Foo version", user=self.user) - foo.save() - - # Give it a single validly indexed pipeline input - foo.create_input(compounddatatype=self.DNAinput_cdt, - dataset_name="oneinput", - dataset_idx=1) - - # Add a valid step 1, but reference itself as the transformation - step1 = foo.steps.create(transformation=foo, step_num=1) - self.assertRaisesRegexp( - ValidationError, - "Step 1 contains the parent pipeline", - step1.clean) - - def test_pipelineStep_oneStep_cabling_referenced_pipeline_references_parent_clean(self): - """Bad step: pipeline step contains the parent pipeline in its lone recursive sub-step.""" - # Define pipeline 'foo' - foo = Pipeline(family=self.DNAcomp_pf, revision_name="foo", revision_desc="Foo version", user=self.user) - foo.save() - - # Give it a single validly indexed pipeline input - foo.create_input(compounddatatype=self.DNAinput_cdt, - dataset_name="oneinput", - dataset_idx=1) - - # Define step 1 as executing DNAcompv2_m - step1 = foo.steps.create(transformation=self.DNAcompv2_m, - step_num=1) - - # Map the input at stpe 1 from Pipeline input "oneinput" - step1.cables_in.create( - dest=step1.transformation.inputs.get(dataset_name="input"), - source_step=0, - source=foo.inputs.get(dataset_name="oneinput")) - - # Define pipeline output at index 1 from (step 1, output "output") - foo.create_outcable( - output_name="oneoutput", - output_idx=1, - source_step=1, - source=step1.transformation.outputs.get(dataset_name="output")) - foo.create_outputs() - foo.save() - - # Define a second pipeline - bar = Pipeline(family=self.DNAcomp_pf, revision_name="bar", revision_desc="Bar version", user=self.user) - bar.save() - - # Give it a single validly indexed pipeline input - bar.create_input(compounddatatype=self.DNAinput_cdt, - dataset_name="barinput", - dataset_idx=1) - - # At step 1, execute the transformation 'foo' defined above - bstep1 = bar.steps.create(transformation=foo, - step_num=1) - - # Map to foo.input "oneinput" from bar pipeline output "barinput" - bstep1.cables_in.create( - dest=foo.inputs.get(dataset_name="oneinput"), - source_step=0, - source=bar.inputs.get(dataset_name="barinput")) - - # Map a single output, from step 1 foo.output = "oneoutput" - bar.create_outcable( - output_name="baroutput", - output_idx=1, - source_step=1, - source=bstep1.transformation.outputs.get(dataset_name="oneoutput")) - bar.save() - - # Now refine foo's step 1 to point to bar - step1.delete() - foo.outputs.all().delete() - - # Have step 1 of foo point to bar (But bar points to foo!) - badstep = foo.steps.create(transformation=bar, - step_num=1) - - self.assertRaisesRegexp( - ValidationError, - "Step 1 contains the parent pipeline", - badstep.clean) - - def test_pipelineStep_manySteps_cabling_referenced_pipeline_references_parent_clean(self): - """Bad step: pipeline step contains the parent pipeline in some recursive sub-step.""" - - # foo invokes DNAcompv2_m at step 1 - foo = Pipeline(family=self.DNAcomp_pf, revision_name="foo", revision_desc="Foo version", user=self.user) - foo.save() - foo.create_input(compounddatatype=self.DNAinput_cdt, - dataset_name="oneinput", - dataset_idx=1) - step1 = foo.steps.create(transformation=self.DNAcompv2_m, - step_num=1) - step1.cables_in.create( - dest=step1.transformation.inputs.get(dataset_name="input"), - source_step=0, - source=foo.inputs.get(dataset_name="oneinput")) - foo.create_outcable( - output_name="oneoutput", output_idx=1, - source_step=1, - source=step1.transformation.outputs.get(dataset_name="output")) - foo.create_outputs() - foo.save() - - # bar invokes foo at step 1 and DNArecomp_m at step 2 - bar = Pipeline(family=self.DNAcomp_pf, revision_name="bar", revision_desc="Bar version", user=self.user) - bar.save() - bar.create_input(compounddatatype=self.DNAinput_cdt, - dataset_name="barinput", - dataset_idx=1) - bstep1 = bar.steps.create(transformation=foo, - step_num=1) - - bstep1.cables_in.create( - dest=bstep1.transformation.inputs.get(dataset_name="oneinput"), - source_step=0, - source=bar.inputs.get(dataset_name="barinput")) - - bstep2 = bar.steps.create(transformation=self.DNArecomp_m, - step_num=2) - bstep2.cables_in.create( - dest=bstep2.transformation.inputs.get(dataset_name="complemented_seqs"), - source_step=1, - source=bstep1.transformation.outputs.get(dataset_name="oneoutput")) - bar.create_outcable( - output_name="baroutputone", - output_idx=1, - source_step=1, - source=bstep1.transformation.outputs.get(dataset_name="oneoutput")) - bar.create_outcable( - output_name="baroutputtwo", - output_idx=2, - source_step=2, - source=bstep2.transformation.outputs.get(dataset_name="recomplemented_seqs")) - bar.save() - - # foo is redefined to be circular - step1.delete() - foo.outputs.all().delete() - badstep = foo.steps.create(transformation=bar, - step_num=1) - self.assertRaisesRegexp( - ValidationError, - "Step 1 contains the parent pipeline", - badstep.clean) - - def test_pipelinestep_outputs_to_delete(self): - """ - Make sure marking an output for deletion actually does so. - """ - step = self.DNAcompv1_p.steps.first() - output = step.transformation.outputs.first() - step.add_deletion(output) - self.assertEqual(len(step.outputs_to_retain()), 0) - self.assertEqual(step.outputs_to_delete.count(), 1) - step.outputs_to_delete.remove(output) - self.assertEqual(len(step.outputs_to_retain()), 1) - self.assertEqual(step.outputs_to_delete.count(), 0) - - def test_delete_pipeline_step(self): - """Deleting a PipelineStep is possible.""" - PipelineStep.objects.first().delete() - - -class PipelineStepInputCableTests(PipelineTestCase): - def test_delete_pipeline_step_input_cable(self): - """Deleting a PipelineStepInputCable is possible.""" - PipelineStepInputCable.objects.first().delete() - - -class PipelineOutputCableTests(PipelineTestCase): - def test_delete_pipeline_output_cable(self): - """Deleting a PipelineOutputCable is possible.""" - PipelineOutputCable.objects.first().delete() - - def test_pipeline_trivial_cable(self): - """ - A trivial cable should have is_trivial() = True. - """ - outcable = self.DNAcompv1_p.outcables.first() # only one outcable - self.assertEqual(outcable.is_trivial(), True) - - -class PipelineStepRawDeleteTests(PipelineTestCase): - - def test_PipelineStep_clean_raw_output_to_be_deleted_good(self): - # Define a single raw input, and a raw + CSV (self.triplet_cdt) output for self.script_4_1_M - self.script_4_1_M.inputs.all().delete() - self.script_4_1_M.create_input(dataset_name="a_b_c", dataset_idx=1) - self.script_4_1_M.create_output( - compounddatatype=self.triplet_cdt, - dataset_name="a_b_c_squared", - dataset_idx=1) - raw_output = self.script_4_1_M.create_output(dataset_name="a_b_c_squared_raw", dataset_idx=2) - self.script_4_1_M.clean() - - # Define 1-step pipeline with a single raw pipeline input - pipeline_1 = self.test_PF.members.create(revision_name="foo", revision_desc="Foo version", user=self.user) - pipeline_1.create_input(dataset_name="a_b_c_pipeline", dataset_idx=1) - step1 = pipeline_1.steps.create(transformation=self.script_4_1_M, step_num=1) - - step1.add_deletion(raw_output) - - self.assertEquals(step1.clean(), None) - self.assertEquals(pipeline_1.clean(), None) - - def test_PipelineStep_clean_delete_single_existent_raw_to_good(self): - # Define a single raw output for self.script_4_1_M - raw_output = self.script_4_1_M.create_output( - dataset_name="a_b_c_squared_raw", dataset_idx=1) - - # Define 1-step pipeline - pipeline_1 = self.test_PF.members.create(revision_name="v1", revision_desc="First version", user=self.user) - step1 = pipeline_1.steps.create(transformation=self.script_4_1_M, step_num=1) - - step1.add_deletion(raw_output) - - self.assertEquals(step1.clean(), None) - - def test_PipelineStep_clean_delete_non_existent_tro_bad(self): - # Define a 1-step pipeline containing self.script_4_1_M which has a raw_output - self.script_4_1_M.create_output(dataset_name="a_b_c_squared_raw", dataset_idx=1) - pipeline_1 = self.test_PF.members.create(revision_name="v1", revision_desc="First version", user=self.user) - step1 = pipeline_1.steps.create(transformation=self.script_4_1_M, step_num=1) - - # Define a 1-step pipeline containing self.script_4_2_M which has a raw_output - self.script_4_2_M = Method(revision_name="s42", - revision_desc="s42", - family=self.test_MF, - driver=self.script_4_1_CRR, - user=self.user) - self.script_4_2_M.save() - raw_output_unrelated = self.script_4_2_M.create_output( - dataset_name="a_b_c_squared_raw", - dataset_idx=1) - pipeline_unrelated = self.test_PF.members.create(revision_name="foo", revision_desc="Foo version", - user=self.user) - pipeline_unrelated.steps.create(transformation=self.script_4_2_M, step_num=1) - - # For pipeline 1, mark a raw output to be deleted in an unrelated method - step1.add_deletion(raw_output_unrelated) - - errorMessage = 'Transformation at step 1 does not have output "1: a_b_c_squared_raw"' - self.assertRaisesRegexp(ValidationError, errorMessage, step1.clean) - self.assertRaisesRegexp(ValidationError, errorMessage, pipeline_1.clean) - - def test_PipelineStep_clean_raw_output_to_be_deleted_in_different_pipeline_bad(self): - # Define a single raw input, and a raw + CSV (self.triplet_cdt) output for self.script_4_1_M - self.script_4_1_M.inputs.all().delete() - self.script_4_1_M.create_input(dataset_name="a_b_c", dataset_idx=1) - self.script_4_1_M.create_output(compounddatatype=self.triplet_cdt, - dataset_name="a_b_c_squared", - dataset_idx=1) - self.script_4_1_M.create_output(dataset_name="a_b_c_squared_raw", - dataset_idx=2) - - self.script_4_2_M = Method(revision_name="s42", - revision_desc="s42", - family=self.test_MF, - driver=self.script_4_1_CRR, - user=self.user) - self.script_4_2_M.save() - unrelated_raw_output = self.script_4_2_M.create_output( - dataset_name="unrelated_raw_output", - dataset_idx=1) - - # Define 1-step pipeline with a single raw pipeline input - pipeline_1 = self.test_PF.members.create(revision_name="foo", revision_desc="Foo version", user=self.user) - pipeline_1.create_input(dataset_name="a_b_c_pipeline", dataset_idx=1) - step1 = pipeline_1.steps.create(transformation=self.script_4_1_M, step_num=1) - - # Define second 1-step pipeline with a single raw pipeline input - pipeline_2 = self.test_PF.members.create(revision_name="bar", - revision_desc="Bar version", - user=self.user) - - pipeline_2.create_input(dataset_name="a_b_c_pipeline", dataset_idx=1) - pipeline_2.steps.create(transformation=self.script_4_2_M, step_num=1) - - # For pipeline 1, mark a raw output to be deleted in a different pipeline (pipeline_2) - step1.add_deletion(unrelated_raw_output) - - error_msg = 'Transformation at step 1 does not have output "1: unrelated_raw_output"' - self.assertRaisesRegexp(ValidationError, error_msg, step1.clean) - self.assertRaisesRegexp(ValidationError, error_msg, pipeline_1.clean) - - -class RawOutputCableTests(PipelineTestCase): - - def test_PipelineOutputCable_raw_outcable_references_valid_step_good(self): - - # Define a single raw input, and a raw + CSV (self.triplet_cdt) output for self.script_4_1_M - self.script_4_1_M.inputs.all().delete() - self.script_4_1_M.outputs.all().delete() - self.script_4_1_M.create_input(dataset_name="a_b_c", dataset_idx=1) - self.script_4_1_M.create_output(compounddatatype=self.triplet_cdt, - dataset_name="a_b_c_squared", - dataset_idx=1) - raw_output = self.script_4_1_M.create_output( - dataset_name="a_b_c_squared_raw", - dataset_idx=2) - - # Define 1-step pipeline with a single raw pipeline input - self.pipeline_1 = self.test_PF.members.create(revision_name="v1", revision_desc="First version", - user=self.user) - self.pipeline_1.create_input(dataset_name="a_b_c_pipeline", - dataset_idx=1) - step1 = self.pipeline_1.steps.create(transformation=self.script_4_1_M, - step_num=1) - - # Outmap a raw cable from a valid step - outcable1 = self.pipeline_1.create_raw_outcable( - raw_output_name="validName", - raw_output_idx=1, - source_step=1, - source=raw_output) - - # Note: pipeline + pipeline step 1 complete_clean would fail (not all inputs are quenched) - self.pipeline_1.create_outputs() - self.assertEquals(step1.clean(), None) - self.assertEquals(outcable1.clean(), None) - self.assertEquals(self.pipeline_1.clean(), None) - - def test_PipelineOutputCable_raw_outcable_references_deleted_output_good(self): - - # Define a single raw input, and a raw + CSV (self.triplet_cdt) output for self.script_4_1_M - self.script_4_1_M.inputs.all().delete() - self.script_4_1_M.outputs.all().delete() - self.script_4_1_M.create_input(dataset_name="a_b_c", dataset_idx=1) - self.script_4_1_M.create_output(compounddatatype=self.triplet_cdt, - dataset_name="a_b_c_squared", - dataset_idx=1) - raw_output = self.script_4_1_M.create_output(dataset_name="a_b_c_squared_raw", - dataset_idx=2) - - # Define 2-step pipeline with a single raw pipeline input - pipeline_1 = self.test_PF.members.create(revision_name="v1", - revision_desc="First version", - user=self.user) - pipeline_1.create_input(dataset_name="a_b_c_pipeline", dataset_idx=1) - step1 = pipeline_1.steps.create(transformation=self.script_4_1_M, - step_num=1) - pipeline_1.steps.create(transformation=self.script_4_1_M, - step_num=2) - - # Outmap a raw cable from a valid step + valid output - outcable1 = pipeline_1.create_raw_outcable(raw_output_name="validName", - raw_output_idx=1, - source_step=1, - source=raw_output) - - # It's not actually deleted yet - so no error - self.assertEquals(outcable1.clean(), None) - - # Mark raw output of step1 as deleted - step1.add_deletion(raw_output) - - # Now it's deleted. - # NOTE August 23, 2013: this doesn't break anymore. - self.assertEquals(outcable1.clean(), None) - self.assertEquals(pipeline_1.clean(), None) - self.assertEquals(step1.clean(), None) - - def test_PipelineOutputCable_raw_outcable_references_valid_step_but_invalid_raw_TO_bad(self): - - # Define 1 raw input, and 1 raw + 1 CSV (self.triplet_cdt) output for method self.script_4_1_M - self.script_4_1_M.inputs.all().delete() - self.script_4_1_M.outputs.all().delete() - self.script_4_1_M.create_input(dataset_name="a_b_c", dataset_idx=1) - self.script_4_1_M.create_output(compounddatatype=self.triplet_cdt, - dataset_name="a_b_c_squared", - dataset_idx=1) - self.script_4_1_M.create_output(dataset_name="a_b_c_squared_raw", - dataset_idx=2) - - # Define an unrelated method and give it a raw output - unrelated_method = Method( - revision_name="s4 - unrelated", - revision_desc="s4 - unrelated", - family=self.test_MF, - driver=self.script_4_1_CRR, - user=self.user - ) - unrelated_method.save() - unrelated_method.clean() - unrelated_raw_output = unrelated_method.create_output( - dataset_name="unrelated_raw_output", - dataset_idx=1) - - # Define 1-step pipeline with a single raw pipeline input - self.pipeline_1 = self.test_PF.members.create(revision_name="v1", revision_desc="First version", - user=self.user) - self.pipeline_1.create_input(dataset_name="a_b_c_pipeline", - dataset_idx=1) - self.pipeline_1.steps.create(transformation=self.script_4_1_M, - step_num=1) - - # Outmap a raw cable to a valid step but a TransformationRawOutput that does not exist at the specified PS - outcable1 = self.pipeline_1.outcables.create( - output_name="validName", - output_idx=1, - source_step=1, - source=unrelated_raw_output) - - self.assertRaisesRegexp( - ValidationError, - 'Transformation at step 1 does not produce output "{}"'.format(unrelated_raw_output), - outcable1.clean) - - def test_PipelineOutputCable_raw_outcable_references_invalid_step_bad(self): - - # Define 1 raw input, and 1 raw + 1 CSV (self.triplet_cdt) output for method self.script_4_1_M - self.script_4_1_M.inputs.all().delete() - self.script_4_1_M.outputs.all().delete() - self.script_4_1_M.create_input(dataset_name="a_b_c", dataset_idx=1) - self.script_4_1_M.create_output(compounddatatype=self.triplet_cdt, - dataset_name="a_b_c_squared", - dataset_idx=1) - raw_output = self.script_4_1_M.create_output(dataset_name="a_b_c_squared_raw", - dataset_idx=2) - self.script_4_1_M.clean() - - # Define 1-step pipeline with a single raw pipeline input - self.pipeline_1 = self.test_PF.members.create(revision_name="v1", - revision_desc="First version", - user=self.user) - self.pipeline_1.create_input(dataset_name="a_b_c_pipeline", dataset_idx=1) - self.pipeline_1.steps.create(transformation=self.script_4_1_M, step_num=1) - - # Outmap a raw cable to an invalid step - outcable1 = self.pipeline_1.outcables.create( - output_name="validName", - output_idx=1, - source_step=2, - source=raw_output) - - error_msg = "Output requested from a non-existent step" - - self.assertRaisesRegexp(ValidationError, error_msg, outcable1.clean) - self.assertRaisesRegexp(ValidationError, error_msg, self.pipeline_1.clean) - self.assertRaisesRegexp(ValidationError, error_msg, - self.pipeline_1.complete_clean) - - -class PipelineUpdateTests(PipelineTestCase): - def create_dependency_revision(self): - """ Find a dependency that is used in a pipeline. - - It should only have a single revision. - Add a second revision for it. - """ - used_methods = Method.objects.filter(pipelinesteps__isnull=False) - dependencies = MethodDependency.objects.filter(method__in=used_methods) - dependency = dependencies.earliest('id') # dependency used in a pipeline - code_resource_revision = dependency.requirement - fn = "GoodRNANucSeq.csv" - with open(os.path.join(samplecode_path, fn), "rb") as f: - new_revision = CodeResourceRevision( - coderesource=code_resource_revision.coderesource, - revision_name="rna", - revision_desc="Switch to RNA", - revision_parent=code_resource_revision, - content_file=File(f), - user=self.myUser) - new_revision.full_clean() - new_revision.save() - new_revision.grant_everyone_access() - - return new_revision - - def create_code_revision(self): - # Define compv2_crRev for comp_cr - fn = "complement_v2.py" - with open(os.path.join(samplecode_path, fn), "rb") as f: - self.compv3_crRev = CodeResourceRevision( - coderesource=self.comp_cr, - revision_name="v3", - revision_desc="Third version: rounder", - revision_parent=self.compv2_crRev, - content_file=File(f), - user=self.myUser) - # case.compv2_crRev.content_file.save(fn, File(f)) - self.compv3_crRev.full_clean() - self.compv3_crRev.save() - self.compv3_crRev.grant_everyone_access() - - def create_method(self): - self.create_code_revision() - self.DNAcompv3_m = self.DNAcomp_mf.members.create( - revision_name="v3", - revision_desc="Third version", - revision_parent=self.DNAcompv2_m, - driver=self.compv3_crRev, - user=self.myUser) - self.DNAcompv3_m.full_clean() - self.DNAcompv3_m.save() - self.DNAcompv3_m.grant_everyone_access() - self.DNAcompv3_m.copy_io_from_parent() - - def test_find_update_not_found(self): - pipeline = self.DNAcomp_pf.members.get(revision_number=2) - update = pipeline.find_update() - - self.assertEqual(update, None) - - def test_find_update(self): - pipeline = self.DNAcomp_pf.members.get(revision_number=1) - next_pipeline = self.DNAcomp_pf.members.get(revision_number=2) - - update = pipeline.find_update() - - self.assertEqual(update, next_pipeline) - - def test_find_step_updates_none(self): - updates = self.DNAcompv1_p.find_step_updates() - - self.assertListEqual(updates, []) - - def test_find_step_updates_method(self): - self.create_method() - - updates = self.DNAcompv1_p.find_step_updates() - - self.assertEqual(len(updates), 1) - update = updates[0] - self.assertEqual(update.step_num, 1) - self.assertEqual(update.method, self.DNAcompv3_m) - self.assertEqual(update.code_resource_revision, None) - - def test_find_step_updates_code_resource(self): - self.create_code_revision() - - updates = self.DNAcompv1_p.find_step_updates() - - self.assertEqual(len(updates), 1) - update = updates[0] - self.assertEqual(update.code_resource_revision, self.compv3_crRev) - - def test_find_step_updates_dependency(self): - new_revision = self.create_dependency_revision() - - updates = self.DNAcompv1_p.find_step_updates() - - self.assertEqual(len(updates), 1) - update = updates[0] - self.assertEqual(len(update.dependencies), 1) - self.assertEqual(update.dependencies[0], new_revision) - - def test_serialize_step_updates_dependency(self): - new_revision = self.create_dependency_revision() - updates = self.DNAcompv1_p.find_step_updates() - - data = PipelineStepUpdateSerializer( - updates, - many=True, - context=DuckContext() - ).data - - self.assertEqual(len(data), 1) - update = data[0] - self.assertEqual(len(update['dependencies']), 1) - self.assertEqual(update['dependencies'][0]['id'], new_revision.id) - - -class RawInputCableTests(PipelineTestCase): - def test_PSIC_raw_cable_comes_from_pipeline_input_good(self): - """ - Cable is fed from a pipeline input. - """ - - # Define a single raw input, and a raw + CSV (self.triplet_cdt) output for self.script_4_1_M - self.script_4_1_M.inputs.all().delete() - self.script_4_1_M.outputs.all().delete() - self.script_4_1_M.create_input(dataset_name="a_b_c", dataset_idx=1) - self.script_4_1_M.create_output(compounddatatype=self.triplet_cdt, - dataset_name="a_b_c_squared", - dataset_idx=1) - self.script_4_1_M.create_output(dataset_name="a_b_c_squared_raw", - dataset_idx=2) - self.script_4_1_M.clean() - - # Define pipeline with a single raw pipeline input - self.pipeline_1 = self.test_PF.members.create(revision_name="v1", revision_desc="First version", - user=self.user) - self.pipeline_1.create_input(dataset_name="a_b_c_pipeline", - dataset_idx=1) - - # Define 2 identical steps within the pipeline - step1 = self.pipeline_1.steps.create(transformation=self.script_4_1_M, - step_num=1) - step2 = self.pipeline_1.steps.create(transformation=self.script_4_1_M, - step_num=2) - - # Cable the pipeline input into step1's transformation's only raw input hole - rawcable1 = step1.create_raw_cable( - dest=self.script_4_1_M.inputs.get(dataset_name="a_b_c"), - source=self.pipeline_1.inputs.get(dataset_name="a_b_c_pipeline")) - - rawcable2 = step2.create_raw_cable( - dest=self.script_4_1_M.inputs.get(dataset_name="a_b_c"), - source=self.pipeline_1.inputs.get(dataset_name="a_b_c_pipeline")) - - # These raw cables were both cabled from the pipeline input and are valid - self.assertEquals(rawcable1.clean(), None) - self.assertEquals(rawcable2.clean(), None) - - def test_PSIC_raw_cable_leads_to_foreign_pipeline_bad(self): - """ - Destination must belong to a PS Transformation in THIS pipeline. - """ - # Define a single raw input, and a raw + CSV (self.triplet_cdt) output for self.script_4_1_M - self.script_4_1_M.inputs.all().delete() - self.script_4_1_M.outputs.all().delete() - self.script_4_1_M.create_input(dataset_name="a_b_c", dataset_idx=1) - self.script_4_1_M.create_output(compounddatatype=self.triplet_cdt, - dataset_name="a_b_c_squared", - dataset_idx=1) - self.script_4_1_M.create_output(dataset_name="a_b_c_squared_raw", - dataset_idx=2) - self.script_4_1_M.clean() - - # Define two different 1-step pipelines with 1 raw pipeline input - self.pipeline_1 = self.test_PF.members.create(revision_name="v1", - revision_desc="First version", - user=self.user) - self.pipeline_1.create_input(dataset_name="a_b_c_pipeline", - dataset_idx=1) - self.pipeline_1.steps.create(transformation=self.script_4_1_M, step_num=1) - - self.pipeline_2 = self.test_PF.members.create(revision_name="v2", revision_desc="Second version", - user=self.user) - self.pipeline_2.save() - self.pipeline_2.create_input(dataset_name="a_b_c_pipeline", dataset_idx=1) - step1_pipeline_2 = self.pipeline_2.steps.create( - transformation=self.script_4_1_M, - step_num=1) - - # Define a raw cable into Pipeline2step1 from Pipeline1's raw - # inputs (Cross-pipeline contamination!) - rawcable1 = step1_pipeline_2.cables_in.create( - dest=step1_pipeline_2.transformation.inputs.get(dataset_name="a_b_c"), - source_step=0, - source=self.pipeline_1.inputs.get(dataset_name="a_b_c_pipeline")) - - error_msg = "Pipeline does not have input .*" - self.assertRaisesRegexp(ValidationError, error_msg, rawcable1.clean) - self.assertRaisesRegexp(ValidationError, error_msg, - step1_pipeline_2.clean) - self.assertRaisesRegexp(ValidationError, error_msg, - step1_pipeline_2.complete_clean) - self.assertRaisesRegexp(ValidationError, error_msg, - self.pipeline_2.clean) - - def test_PSIC_raw_cable_does_not_map_to_raw_input_of_this_step_bad(self): - """ - dest does not specify a TransformationRawInput of THIS pipeline step - """ - # Define a single raw input, and a raw + CSV (self.triplet_cdt) output for self.script_4_1_M - self.script_4_1_M.inputs.all().delete() - self.script_4_1_M.outputs.all().delete() - self.script_4_1_M.create_input(dataset_name="a_b_c_method", - dataset_idx=1) - self.script_4_1_M.create_output(compounddatatype=self.triplet_cdt, - dataset_name="a_b_c_squared", - dataset_idx=1) - self.script_4_1_M.create_output(dataset_name="a_b_c_squared_raw", - dataset_idx=2) - - # Define second unrelated method not part of any pipeline but containing a raw input with the same name (a_b_c) - self.script_4_2_M = Method( - revision_name="s4", - revision_desc="s4", - family=self.test_MF, - driver=self.script_4_1_CRR, - user=self.user - ) - self.script_4_2_M.save() - self.script_4_2_M.create_input(dataset_name="a_b_c_method", dataset_idx=1) - - # Define pipeline with a single raw pipeline input and a single step - self.pipeline_1 = self.test_PF.members.create(revision_name="v1", revision_desc="First version", - user=self.user) - self.pipeline_1.create_input(dataset_name="a_b_c_pipeline", dataset_idx=1) - step1 = self.pipeline_1.steps.create(transformation=self.script_4_1_M, - step_num=1) - - # Cable the pipeline input into a raw input hole but from an irrelevent method - rawcable1 = step1.cables_in.create( - dest=self.script_4_2_M.inputs.get(dataset_name="a_b_c_method"), - source_step=0, - source=self.pipeline_1.inputs.get(dataset_name="a_b_c_pipeline")) - - error_msg = 'Transformation at step 1 does not have input "{}"'.format(rawcable1.dest) - self.assertRaisesRegexp(ValidationError, error_msg, rawcable1.clean) - self.assertRaisesRegexp(ValidationError, error_msg, step1.clean) - self.assertRaisesRegexp(ValidationError, error_msg, step1.complete_clean) - self.assertRaisesRegexp(ValidationError, error_msg, self.pipeline_1.clean) - self.assertRaisesRegexp(ValidationError, error_msg, self.pipeline_1.complete_clean) - - def test_PSIC_raw_cable_has_custom_wiring_defined(self): - """ - Raw PSIC has custom wiring defined. - """ - # Define a single raw input, and a raw + CSV (self.triplet_cdt) output for self.script_4_1_M - self.script_4_1_M.inputs.all().delete() - self.script_4_1_M.outputs.all().delete() - self.script_4_1_M.create_input(dataset_name="a_b_c", dataset_idx=1) - self.script_4_1_M.create_output(compounddatatype=self.triplet_cdt, - dataset_name="a_b_c_squared", - dataset_idx=1) - self.script_4_1_M.create_output(dataset_name="a_b_c_squared_raw", - dataset_idx=2) - self.script_4_1_M.clean() - - # Define pipeline with a single raw pipeline input - self.pipeline_1 = self.test_PF.members.create(revision_name="v1", revision_desc="First version", - user=self.user) - self.pipeline_1.create_input(dataset_name="a_b_c_pipeline", - dataset_idx=1) - - # Define 2 identical steps within the pipeline - step1 = self.pipeline_1.steps.create(transformation=self.script_4_1_M, - step_num=1) - step2 = self.pipeline_1.steps.create(transformation=self.script_4_1_M, - step_num=2) - - # Cable the pipeline input into step1's transformation's only raw input hole - rawcable1 = step1.create_raw_cable( - dest=self.script_4_1_M.inputs.get(dataset_name="a_b_c"), - source=self.pipeline_1.inputs.get(dataset_name="a_b_c_pipeline")) - - step2.create_raw_cable( - dest=self.script_4_1_M.inputs.get(dataset_name="a_b_c"), - source=self.pipeline_1.inputs.get(dataset_name="a_b_c_pipeline")) - - # Define custom wiring (totally nonsensical) on rawcable1. - rawcable1.custom_wires.create( - source_pin=self.doublet_cdt.members.all()[0], - dest_pin=self.doublet_cdt.members.all()[0]) - - self.assertRaisesRegexp( - ValidationError, - re.escape('Cable "{}" is raw and should not have custom wiring defined'.format(rawcable1)), - rawcable1.clean) - - -class RawSaveTests(PipelineTestCase): - def test_method_with_raw_input_defined_do_not_copy_raw_xputs_to_new_revision(self): - # Give script_4_1_M a raw input - self.script_4_1_M.inputs.all().delete() - self.script_4_1_M.create_input(dataset_name="a_b_c", dataset_idx=1) - - # Make a method without a parent - self.script_4_2_M = Method( - revision_name="s4", - revision_desc="s4", - family=self.test_MF, - driver=self.script_4_1_CRR, - user=self.user - ) - self.script_4_2_M.save() - - # There should be no raw inputs/outputs - self.assertEqual(self.script_4_2_M.inputs.count(), 0) - self.assertEqual(self.script_4_2_M.outputs.count(), 0) - - def test_method_with_raw_output_defined_do_not_copy_raw_xputs_to_new_revision(self): - # Give script_4_1_M a raw output - self.script_4_1_M.create_output(dataset_name="a_b_c", dataset_idx=1) - - # Make a method without a parent - self.script_4_2_M = Method(revision_name="s4", - revision_desc="s4", - family=self.test_MF, - driver=self.script_4_1_CRR, - user=self.user) - self.script_4_2_M.save() - - # There should be no raw inputs/outputs - self.assertEqual(self.script_4_2_M.inputs.count(), 0) - self.assertEqual(self.script_4_2_M.outputs.count(), 0) - - def test_method_with_no_xputs_defined_copy_raw_xputs_to_new_revision(self): - - # Give script_4_1_M a raw input - self.script_4_1_M.inputs.all().delete() - self.script_4_1_M.create_input(dataset_name="a_b_c", dataset_idx=1) - - # Make a method with a parent, and do not specify inputs/outputs - self.script_4_2_M = Method( - revision_parent=self.script_4_1_M, - revision_name="s4", - revision_desc="s4", - family=self.test_MF, - driver=self.script_4_1_CRR, - user=self.user) - self.script_4_2_M.save() - self.script_4_2_M.copy_io_from_parent() - - # The input should have been copied over (SUBOPTIMAL TEST) - self.assertEqual(self.script_4_1_M.inputs.all()[0].dataset_name, - self.script_4_2_M.inputs.all()[0].dataset_name) - self.assertEqual(self.script_4_1_M.inputs.all()[0].dataset_idx, - self.script_4_2_M.inputs.all()[0].dataset_idx) - - -# August 23, 2013: these are kind of redundant now but what the hey. -class SingleRawInputTests(PipelineTestCase): - def test_transformation_rawinput_coexists_with_nonraw_inputs_clean_good(self): - - # Define raw input "a_b_c" at index = 1 - self.script_4_1_M.inputs.all().delete() - self.script_4_1_M.create_input(dataset_name="a_b_c", dataset_idx=1) - - # Define input "a_b_c_squared" of type "triplet_cdt" at index = 2 - self.script_4_1_M.create_input( - compounddatatype=self.triplet_cdt, - dataset_name="a_b_c_squared", - dataset_idx=2) - self.script_4_1_M.save() - - # Neither the names nor the indices conflict - this should pass - self.assertEquals(self.script_4_1_M.check_input_indices(), None) - self.assertEquals(self.script_4_1_M.check_output_indices(), None) - self.assertEquals(self.script_4_1_M.clean(), None) - - def test_transformation_rawinput_coexists_with_nonraw_inputs_but_not_consecutive_indexed_bad(self): - - # Define raw input "a_b_c" at index = 1 - self.script_4_1_M.inputs.all().delete() - self.script_4_1_M.create_input(dataset_name="a_b_c", dataset_idx=1) - - # Define input name "a_b_c_squared" of type "triplet_cdt" at nonconsecutive index = 3 - self.script_4_1_M.create_input(compounddatatype=self.triplet_cdt, - dataset_name="a_b_c_squared", - dataset_idx=3) - self.script_4_1_M.save() - - # The indices are not consecutive - self.assertRaisesRegexp( - ValidationError, - "Inputs are not consecutively numbered starting from 1", - self.script_4_1_M.check_input_indices) - - self.assertRaisesRegexp( - ValidationError, - "Inputs are not consecutively numbered starting from 1", - self.script_4_1_M.clean) - - def test_PipelineStep_completeClean_check_quenching_of_raw_inputs_good(self): - # Wire 1 raw input to a pipeline step that expects only 1 input - self.script_4_1_M.inputs.all().delete() - method_raw_in = self.script_4_1_M.create_input(dataset_name="a_b_c", - dataset_idx=1) - - # Define 1-step pipeline with a single raw pipeline input - self.pipeline_1 = self.test_PF.members.create(revision_name="v1", revision_desc="First version", - user=self.user) - pipeline_input = self.pipeline_1.create_input(dataset_name="a_b_c_pipeline", - dataset_idx=1) - step1 = self.pipeline_1.steps.create(transformation=self.script_4_1_M, - step_num=1) - - step1.create_raw_cable( - dest=method_raw_in, - source=pipeline_input) - - self.assertEquals(step1.clean(), None) - self.assertEquals(step1.complete_clean(), None) - - def test_PipelineStep_completeClean_check_overquenching_doubled_source_of_raw_inputs_bad(self): - - # Wire 1 raw input to a pipeline step that expects only 1 input - self.script_4_1_M.inputs.all().delete() - method_raw_in = self.script_4_1_M.create_input(dataset_name="a_b_c", - dataset_idx=1) - - # Define 1-step pipeline with a single raw pipeline input - self.pipeline_1 = self.test_PF.members.create(revision_name="v1", revision_desc="First version", - user=self.user) - pipeline_input = self.pipeline_1.create_input(dataset_name="a_b_c_pipeline", - dataset_idx=1) - step1 = self.pipeline_1.steps.create(transformation=self.script_4_1_M, - step_num=1) - - step1.create_raw_cable( - dest=method_raw_in, - source=pipeline_input) - - step1.create_raw_cable( - dest=method_raw_in, - source=pipeline_input) - - errorMessage = "Input \"a_b_c\" to transformation at step 1 is cabled more than once" - self.assertRaisesRegexp( - ValidationError, - errorMessage, - step1.clean) - - self.assertRaisesRegexp( - ValidationError, - errorMessage, - step1.complete_clean) - - def test_PipelineStep_completeClean_check_overquenching_different_sources_of_raw_inputs_bad(self): - # Wire 1 raw input to a pipeline step that expects only 1 input - self.script_4_1_M.inputs.all().delete() - method_raw_in = self.script_4_1_M.create_input(dataset_name="a_b_c", - dataset_idx=1) - - # Define 1-step pipeline with a single raw pipeline input - self.pipeline_1 = self.test_PF.members.create(revision_name="v1", revision_desc="First version", - user=self.user) - pipeline_input = self.pipeline_1.create_input(dataset_name="a_b_c_pipeline", - dataset_idx=1) - pipeline_input_2 = self.pipeline_1.create_input(dataset_name="a_b_c_pipeline_2", - dataset_idx=2) - - step1 = self.pipeline_1.steps.create(transformation=self.script_4_1_M, - step_num=1) - - step1.create_raw_cable( - dest=method_raw_in, - source=pipeline_input) - - step1.create_raw_cable( - dest=method_raw_in, - source=pipeline_input_2) - - errorMessage = "Input \"a_b_c\" to transformation at step 1 is cabled more than once" - self.assertRaisesRegexp(ValidationError, errorMessage, step1.clean) - self.assertRaisesRegexp( - ValidationError, - errorMessage, - step1.complete_clean) - - def test_PipelineStep_completeClean_check_underquenching_of_raw_inputs_bad(self): - # Wire 1 raw input to a pipeline step that expects only 1 input - self.script_4_1_M.inputs.all().delete() - self.script_4_1_M.create_input(dataset_name="a_b_c", dataset_idx=1) - - # Define 1-step pipeline with a single raw pipeline input - self.pipeline_1 = self.test_PF.members.create(revision_name="v1", revision_desc="First version", - user=self.user) - step1 = self.pipeline_1.steps.create(transformation=self.script_4_1_M, step_num=1) - - errorMessage = "Input \"a_b_c\" to transformation at step 1 is not cabled'" - self.assertEquals(step1.clean(), None) - self.assertRaisesRegexp( - ValidationError, - errorMessage, - step1.complete_clean) - - -class SeveralRawInputsTests(PipelineTestCase): - def test_transformation_several_rawinputs_coexists_with_several_nonraw_inputs_clean_good(self): - # Note that this method wouldn't actually run -- inputs don't match. - - self.script_4_1_M.inputs.all().delete() - # Define raw input "a_b_c" at index = 1 - self.script_4_1_M.create_input(dataset_name="a_b_c", dataset_idx=1) - - # Define raw input "RawIn3" at index = 3 - self.script_4_1_M.create_input(dataset_name="RawIn3", dataset_idx=3) - - # Define input "a_b_c_squared" of type "triplet_cdt" at index = 2 - self.script_4_1_M.create_input(compounddatatype=self.triplet_cdt, - dataset_name="a_b_c_squared", - dataset_idx=2) - - # Define input "Input4" of type "doublet_cdt" at index = 4 - self.script_4_1_M.create_input(compounddatatype=self.doublet_cdt, - dataset_name="Input4", - dataset_idx=4) - - # Neither the names nor the indices conflict - this should pass - self.assertEquals(self.script_4_1_M.check_input_indices(), None) - self.assertEquals(self.script_4_1_M.check_output_indices(), None) - self.assertEquals(self.script_4_1_M.clean(), None) - - def test_transformation_several_rawinputs_several_nonraw_inputs_not1based_bad(self): - # Note that this method wouldn't actually run -- inputs don't match. - self.script_4_1_M.inputs.all().delete() - - # Define raw input "a_b_c" at index = 2 - self.script_4_1_M.create_input(dataset_name="a_b_c", dataset_idx=2) - - # Define raw input "RawIn3" at index = 3 - self.script_4_1_M.create_input(dataset_name="RawIn3", dataset_idx=3) - - # Define input "a_b_c_squared" of type "triplet_cdt" at index = 4 - self.script_4_1_M.create_input(compounddatatype=self.triplet_cdt, - dataset_name="a_b_c_squared", - dataset_idx=4) - - # Define input "Input4" of type "doublet_cdt" at index = 5 - self.script_4_1_M.create_input(compounddatatype=self.doublet_cdt, - dataset_name="Input4", - dataset_idx=5) - - self.assertRaisesRegexp( - ValidationError, - "Inputs are not consecutively numbered starting from 1", - self.script_4_1_M.check_input_indices) - self.assertEquals(self.script_4_1_M.check_output_indices(), None) - self.assertRaisesRegexp( - ValidationError, - "Inputs are not consecutively numbered starting from 1", - self.script_4_1_M.clean) - - def test_transformation_several_rawinputs_several_nonraw_inputs_nonconsecutive_bad(self): - # Note that this method wouldn't actually run -- inputs don't match. - - # Define raw input "a_b_c" at index = 2 - self.script_4_1_M.create_input(dataset_name="a_b_c", dataset_idx=2) - - # Define raw input "RawIn3" at index = 3 - self.script_4_1_M.create_input(dataset_name="RawIn3", dataset_idx=3) - - # Define input "a_b_c_squared" of type "triplet_cdt" at index = 5 - self.script_4_1_M.create_input(compounddatatype=self.triplet_cdt, - dataset_name="a_b_c_squared", - dataset_idx=5) - - # Define input "Input4" of type "doublet_cdt" at index = 6 - self.script_4_1_M.create_input(compounddatatype=self.doublet_cdt, - dataset_name="Input6", - dataset_idx=6) - - self.assertRaisesRegexp( - ValidationError, - "Inputs are not consecutively numbered starting from 1", - self.script_4_1_M.check_input_indices) - self.assertEquals(self.script_4_1_M.check_output_indices(), None) - self.assertRaisesRegexp( - ValidationError, - "Inputs are not consecutively numbered starting from 1", - self.script_4_1_M.clean) - - def test_pipeline_several_rawinputs_coexists_with_several_nonraw_inputs_clean_good(self): - # Define 1-step pipeline with conflicting inputs - pipeline_1 = self.test_PF.members.create(revision_name="foo", revision_desc="Foo version", - user=self.user) - pipeline_1.create_input(dataset_name="input_1_raw", - dataset_idx=1) - pipeline_1.create_input(compounddatatype=self.triplet_cdt, - dataset_name="input_2", - dataset_idx=2) - pipeline_1.create_input(dataset_name="input_3_raw", - dataset_idx=3) - pipeline_1.create_input(compounddatatype=self.triplet_cdt, - dataset_name="input_4", - dataset_idx=4) - - # Neither the names nor the indices conflict - this should pass - self.assertEquals(pipeline_1.check_input_indices(), None) - self.assertEquals(pipeline_1.clean(), None) - - # We consider this enough for the multiple input case, as the - # single case was thoroughly checked. - - def test_PipelineStep_completeClean_check_overquenching_different_sources_of_raw_inputs_bad(self): - - # Define 2 inputs for the method - self.script_4_1_M.inputs.all().delete() - method_raw_in = self.script_4_1_M.create_input(dataset_name="method_in_1", dataset_idx=1) - method_raw_in_2 = self.script_4_1_M.create_input(dataset_name="method_in_2", dataset_idx=2) - - # Define 1-step pipeline with 2 raw pipeline inputs - self.pipeline_1 = self.test_PF.members.create(revision_name="v1", revision_desc="First version", - user=self.user) - pipeline_input = self.pipeline_1.create_input(dataset_name="a_b_c_pipeline", dataset_idx=1) - pipeline_input_2 = self.pipeline_1.create_input(dataset_name="a_b_c_pipeline_2", dataset_idx=2) - - step1 = self.pipeline_1.steps.create(transformation=self.script_4_1_M, step_num=1) - - step1.create_raw_cable( - dest=method_raw_in, - source=pipeline_input) - - step1.create_raw_cable( - dest=method_raw_in_2, - source=pipeline_input_2) - - step1.create_raw_cable( - dest=method_raw_in, - source=pipeline_input_2) - - errorMessage = "Input \"method_in_1\" to transformation at step 1 is cabled more than once" - self.assertRaisesRegexp( - ValidationError, - errorMessage, - step1.clean) - - self.assertRaisesRegexp( - ValidationError, - errorMessage, - step1.complete_clean) - - -# August 23, 2013: these also seem pretty redundant, but let's just leave 'em. -class SingleRawOutputTests(PipelineTestCase): - def test_transformation_rawoutput_coexists_with_nonraw_outputs_clean_good(self): - - # Define raw output "a_b_c" at index = 1 - self.script_4_1_M.create_output(dataset_name="a_b_c", dataset_idx=1) - - # Define output name "a_b_c_squared" of type "triplet_cdt" at index = 2 - self.script_4_1_M.create_output(compounddatatype=self.triplet_cdt, dataset_name="a_b_c_squared", dataset_idx=2) - self.script_4_1_M.save() - - # Neither the names nor the indices conflict - this should pass - self.assertEquals(self.script_4_1_M.check_input_indices(), None) - self.assertEquals(self.script_4_1_M.check_output_indices(), None) - self.assertEquals(self.script_4_1_M.clean(), None) - - def test_transformation_rawoutput_coexists_with_nonraw_outputs_but_not_consecutive_indexed_bad(self): - # Define raw output "a_b_c" at index = 1 - self.script_4_1_M.create_output(dataset_name="a_b_c", dataset_idx=1) - - # Define output name "a_b_c" of type "triplet_cdt" at invalid index = 3 - self.script_4_1_M.create_output(compounddatatype=self.triplet_cdt, dataset_name="a_b_c_squared", dataset_idx=3) - self.script_4_1_M.save() - - # The indices are invalid - self.assertRaisesRegexp( - ValidationError, - "Outputs are not consecutively numbered starting from 1", - self.script_4_1_M.check_output_indices) - - self.assertRaisesRegexp( - ValidationError, - "Outputs are not consecutively numbered starting from 1", - self.script_4_1_M.clean) - - -class SeveralRawOutputsTests(PipelineTestCase): - - def test_transformation_several_rawoutputs_coexists_with_several_nonraw_outputs_clean_good(self): - # Note: the method we define here doesn't correspond to reality; the - # script doesn't have all of these outputs. - - # Define raw output "a_b_c" at index = 1 - self.script_4_1_M.create_output(dataset_name="a_b_c", dataset_idx=1) - - # Define raw output "RawOutput4" at index = 4 - self.script_4_1_M.create_output(dataset_name="RawOutput4", dataset_idx=4) - - # Define output name "foo" of type "doublet_cdt" at index = 3 - self.script_4_1_M.create_output(compounddatatype=self.doublet_cdt, dataset_name="Output3", dataset_idx=3) - - # Define output name "a_b_c_squared" of type "triplet_cdt" at index = 2 - self.script_4_1_M.create_output(compounddatatype=self.triplet_cdt, dataset_name="a_b_c_squared", dataset_idx=2) - - # Neither the names nor the indices conflict - this should pass - self.assertEquals(self.script_4_1_M.check_input_indices(), None) - self.assertEquals(self.script_4_1_M.check_output_indices(), None) - self.assertEquals(self.script_4_1_M.clean(), None) - - def test_transformation_several_rawoutputs_with_several_nonraw_outputs_clean_indices_nonconsecutive_bad(self): - # Note: the method we define here doesn't correspond to reality; the - # script doesn't have all of these outputs. - - # Define raw output "a_b_c" at index = 1 - self.script_4_1_M.create_output(dataset_name="a_b_c", dataset_idx=1) - - # Define raw output "RawOutput4" at index = 2 - self.script_4_1_M.create_output(dataset_name="RawOutput2", dataset_idx=2) - - # Define output name "foo" of type "doublet_cdt" at index = 5 - self.script_4_1_M.create_output(compounddatatype=self.doublet_cdt, dataset_name="Output5", dataset_idx=5) - - # Define output name "a_b_c_squared" of type "triplet_cdt" at index = 10 - self.script_4_1_M.create_output(compounddatatype=self.triplet_cdt, dataset_name="a_b_c_squared", dataset_idx=10) - - # Neither the names nor the indices conflict, but numbering is bad. - self.assertEquals(self.script_4_1_M.check_input_indices(), None) - self.assertRaisesRegexp( - ValidationError, - "Outputs are not consecutively numbered starting from 1", - self.script_4_1_M.check_output_indices) - self.assertRaisesRegexp( - ValidationError, - "Outputs are not consecutively numbered starting from 1", - self.script_4_1_M.clean) - - -class CustomWiringTests(PipelineTestCase): - - def test_CustomCableWire_wires_from_pipeline_input_identical_dt_good(self): - """ Custom wiring that connects identical datatypes together. - - On a cable leading from pipeline input (not PS output). - """ - # Define a pipeline with single pipeline input of type triplet_cdt - my_pipeline = self.test_PF.members.create(revision_name="foo", revision_desc="Foo version", user=self.user) - my_pipeline.inputs.all().delete() - pipeline_in = my_pipeline.create_input( - compounddatatype=self.triplet_cdt, - dataset_name="pipe_in_1", - dataset_idx=1) - - # Define method to have an input with the same CDT, add it as a step, cable it - self.testmethod.inputs.all().delete() - method_in = self.testmethod.create_input( - dataset_name="TestIn", - dataset_idx=1, - compounddatatype=self.triplet_cdt) - my_step1 = my_pipeline.steps.create( - transformation=self.testmethod, step_num=1) - my_cable1 = my_step1.cables_in.create( - dest=method_in, source_step=0, source=pipeline_in) - - # Both CDTs exactly match - self.assertEquals(my_cable1.clean(), None) - self.assertEquals(my_cable1.clean_and_completely_wired(), None) - - # But we can add custom wires anyways - wire1 = my_cable1.custom_wires.create( - source_pin=pipeline_in.get_cdt().members.get(column_idx=1), - dest_pin=method_in.get_cdt().members.get(column_idx=1)) - - # This wire is clean, and the cable is also clean - but not completely wired - self.assertEquals(wire1.clean(), None) - self.assertEquals(my_cable1.clean(), None) - - # It might complain about either connection, so accept either. - self.assertRaisesRegexp( - ValidationError, - 'Destination member "(b: string|c: string)" has no wires leading to it', - my_cable1.clean_and_completely_wired) - - # Here, we wire the remaining 2 CDT members - for i in range(2, 4): - my_cable1.custom_wires.create( - source_pin=pipeline_in.get_cdt().members.get(column_idx=i), - dest_pin=method_in.get_cdt().members.get(column_idx=i)) - - # All the wires are clean - and now the cable is completely wired - for wire in my_cable1.custom_wires.all(): - self.assertEquals(wire.clean(), None) - - self.assertEquals(my_cable1.clean(), None) - self.assertEquals(my_cable1.clean_and_completely_wired(), None) - - def test_CustomCableWire_clean_for_datatype_compatibility(self): - # Wiring test 1 - Datatypes are identical (x -> x) - # Wiring test 2 - Datatypes are not identical, but compatible (y restricts x, y -> x) - # Wiring test 3 - Datatypes are not compatible (z does not restrict x, z -> x) - - # Define 2 CDTs3 datatypes - one identical, one compatible, and - # one incompatible + make a new CDT composed of them - # Regarding datatypes, recall [self.DNA_dt] restricts [self.string_dt] - - # Define a datatype that has nothing to do with anything and have it restrict - # the builtin Shipyard string Datatype. - self.incompatible_dt = Datatype(name="Not compatible", - description="A datatype not having anything to do with anything", - user=self.user) - self.incompatible_dt.save() - self.incompatible_dt.grant_everyone_access() - self.incompatible_dt.restricts.add(Datatype.objects.get(pk=datatypes.STR_PK)) - - # Define 2 CDTs that are unequal: (DNA, string, string), and (string, DNA, incompatible) - cdt_1 = CompoundDatatype(user=self.user) - cdt_1.save() - cdt_1.grant_everyone_access() - cdt_1.members.create(datatype=self.DNA_dt, column_name="col_1", column_idx=1) - cdt_1.members.create(datatype=self.string_dt, column_name="col_2", column_idx=2) - cdt_1.members.create(datatype=self.string_dt, column_name="col_3", column_idx=3) - - cdt_2 = CompoundDatatype(user=self.user) - cdt_2.save() - cdt_2.grant_everyone_access() - cdt_2.members.create(datatype=self.string_dt, column_name="col_1", column_idx=1) - cdt_2.members.create(datatype=self.DNA_dt, column_name="col_2", column_idx=2) - cdt_2.members.create(datatype=self.incompatible_dt, column_name="col_3", column_idx=3) - - # Define a pipeline with single pipeline input of type cdt_1 - my_pipeline = self.test_PF.members.create(revision_name="foo", revision_desc="Foo version", user=self.user) - my_pipeline.grant_everyone_access() - pipeline_in = my_pipeline.create_input(compounddatatype=cdt_1, dataset_name="pipe_in_1", dataset_idx=1) - - # Define method to have an input with cdt_2, add it as a step, cable it - self.testmethod.inputs.all().delete() - method_in = self.testmethod.create_input(dataset_name="TestIn", dataset_idx=1, compounddatatype=cdt_2) - my_step1 = my_pipeline.steps.create(transformation=self.testmethod, step_num=1) - my_cable1 = my_step1.cables_in.create(dest=method_in, source_step=0, source=pipeline_in) - - # CDTs are not equal, so this cable requires custom wiring - self.assertRaisesRegexp( - ValidationError, - 'Custom wiring required for cable "{}"'.format(my_cable1), - my_step1.clean) - - # Wiring case 1: Datatypes are identical (DNA -> DNA) - wire1 = my_cable1.custom_wires.create(source_pin=pipeline_in.get_cdt().members.get(column_idx=1), - dest_pin=method_in.get_cdt().members.get(column_idx=2)) - - # Wiring case 2: Datatypes are compatible (DNA -> string) - wire2 = my_cable1.custom_wires.create(source_pin=pipeline_in.get_cdt().members.get(column_idx=1), - dest_pin=method_in.get_cdt().members.get(column_idx=1)) - - # Wiring case 3: Datatypes are compatible (DNA -> incompatible CDT) - wire3_bad = my_cable1.custom_wires.create(source_pin=pipeline_in.get_cdt().members.get(column_idx=1), - dest_pin=method_in.get_cdt().members.get(column_idx=3)) - - self.assertIsNone(wire1.clean()) - self.assertIsNone(wire2.clean()) - - errorMessage = ('The datatype of the source pin "col_1: DNANucSeq" is incompatible with the datatype of the ' - 'destination pin "col_3: Not compatible"') - - self.assertRaisesRegexp(ValidationError, errorMessage, wire3_bad.clean) - self.assertRaisesRegexp(ValidationError, errorMessage, my_cable1.clean) - - def test_CustomCableWire_clean_source_and_dest_pin_do_not_come_from_cdt_bad(self): - # For source_pin and dest_pin, give a CDTM from an unrelated CDT - - # Define a datatype that has nothing to do with anything. - self.incompatible_dt = Datatype(name="poop", description="poop!!", user=self.user) - self.incompatible_dt.save() - self.incompatible_dt.restricts.add(Datatype.objects.get(pk=datatypes.STR_PK)) - - # Define 2 different CDTs: (DNA, string, string), and (string, DNA, incompatible) - cdt_1 = CompoundDatatype(user=self.user) - cdt_1.save() - cdt_1.members.create(datatype=self.DNA_dt, column_name="col_1", column_idx=1) - cdt_1.members.create(datatype=self.string_dt, column_name="col_2", column_idx=2) - cdt_1.members.create(datatype=self.string_dt, column_name="col_3", column_idx=3) - - cdt_2 = CompoundDatatype(user=self.user) - cdt_2.save() - cdt_2.members.create(datatype=self.string_dt, column_name="col_1", column_idx=1) - cdt_2.members.create(datatype=self.DNA_dt, column_name="col_2", column_idx=2) - cdt_2.members.create(datatype=self.incompatible_dt, column_name="col_3", column_idx=3) - - # Define 2 methods with different inputs - method_1 = Method(revision_name="s4", revision_desc="s4", family=self.test_MF, driver=self.script_4_1_CRR, - user=self.user) - method_1.save() - method_1_in = method_1.create_input(dataset_name="TestIn", dataset_idx=1, compounddatatype=cdt_1) - - method_2 = Method(revision_name="s5", revision_desc="s5", family=self.test_MF, driver=self.script_4_1_CRR, - user=self.user) - method_2.save() - method_2_in = method_2.create_input(dataset_name="TestIn", dataset_idx=1, compounddatatype=cdt_2) - - # Define 2 pipelines - pipeline_1 = self.test_PF.members.create(revision_name="foo", revision_desc="Foo version", user=self.user) - pipeline_1_in = pipeline_1.create_input(compounddatatype=cdt_1, dataset_name="pipe_in_1", dataset_idx=1) - pipeline_1_step = pipeline_1.steps.create(transformation=method_1, step_num=1) - pipeline_1_cable = pipeline_1_step.cables_in.create(dest=method_1_in, source_step=0, source=pipeline_1_in) - - pipeline_2 = self.test_PF.members.create(revision_name="foo", revision_desc="Foo version", user=self.user) - pipeline_2_in = pipeline_2.create_input(compounddatatype=cdt_2, dataset_name="pipe_in_1", dataset_idx=1) - pipeline_2_step = pipeline_2.steps.create(transformation=method_2, step_num=1) - pipeline_2_cable = pipeline_2_step.cables_in.create(dest=method_2_in, source_step=0, source=pipeline_2_in) - - # Within pipeline_1_cable, wire into method 1 idx 1 (Expects DNA) a dest_pin from pipeline 2 idx 3 - # (incompatible dt, cdtm from unrelated cdt) - wire1 = pipeline_1_cable.custom_wires.create(source_pin=pipeline_2_in.get_cdt().members.get(column_idx=3), - dest_pin=method_1_in.get_cdt().members.get(column_idx=1)) - - self.assertRaisesRegexp(ValidationError, - re.escape('Source pin "{}" does not come from compounddatatype "{}"' - .format(wire1.source_pin, cdt_1)), - wire1.clean) - wire1.delete() - - # Within pipeline_1_cable, wire into method 1 idx 1 (Expects DNA) a dest_pin from pipeline 2 idx 1 - # (same dt, cdtm from unrelated cdt) - wire1_alt = pipeline_1_cable.custom_wires.create(source_pin=pipeline_2_in.get_cdt().members.get(column_idx=3), - dest_pin=method_1_in.get_cdt().members.get(column_idx=1)) - - self.assertRaisesRegexp(ValidationError, - re.escape('Source pin "{}" does not come from compounddatatype "{}"' - .format(wire1_alt.source_pin, cdt_1)), - wire1_alt.clean) - - # Try to wire something into cable 2 with a source_pin from cable 1 - wire2 = pipeline_2_cable.custom_wires.create(source_pin=pipeline_1_in.get_cdt().members.get(column_idx=3), - dest_pin=method_2_in.get_cdt().members.get(column_idx=1)) - - self.assertRaisesRegexp(ValidationError, - re.escape('Source pin "{}" does not come from compounddatatype "{}"' - .format(wire2.source_pin, cdt_2)), - wire2.clean) - - -# August 23, 2013: This is pretty redundant now. -class PipelineOutputCableRawTests(PipelineTestCase): - - def test_pipeline_check_for_colliding_outputs_clean_good(self): - - # Define 1-step pipeline with 2 raw pipeline inputs - self.pipeline_1 = self.test_PF.members.create(revision_name="v1", revision_desc="First version", - user=self.user) - self.pipeline_1.create_input(dataset_name="a_b_c_pipeline", dataset_idx=1) - self.pipeline_1.steps.create(transformation=self.script_4_1_M, step_num=1) - - script_4_1_M = self.script_4_1_M - - script_4_1_M.create_output( - compounddatatype=self.mix_triplet_cdt, - dataset_name="scriptOutput1", - dataset_idx=1) - - output_3 = script_4_1_M.create_output( - compounddatatype=self.mix_triplet_cdt, - dataset_name="scriptOutput3", - dataset_idx=3) - - raw_output_2 = script_4_1_M.create_output( - dataset_name="scriptOutput2", - dataset_idx=2) - - raw_output_4 = script_4_1_M.create_output( - dataset_name="scriptOutput4", - dataset_idx=4) - - self.pipeline_1.create_raw_outcable( - raw_output_name="pipeline_output_1", - raw_output_idx=1, - source_step=1, - source=raw_output_2) - - self.pipeline_1.create_raw_outcable( - raw_output_name="pipeline_output_3", - raw_output_idx=3, - source_step=1, - source=raw_output_4) - - self.pipeline_1.create_outcable( - output_name="pipeline_output_2", - output_idx=2, - source_step=1, - source=output_3) - - self.assertEquals(self.pipeline_1.clean(), None) - - -class CustomRawOutputCablingTests(PipelineTestCase): - - def test_Pipeline_create_multiple_raw_outputs_with_raw_outmap(self): - self.my_pipeline = self.test_PF.members.create(revision_name="foo", revision_desc="Foo version", - user=self.user) - - self.my_pipeline.create_input( - compounddatatype=self.triplet_cdt, - dataset_name="pipeline_in_1", - dataset_idx=1) - - # Give the method self.triplet_cdt output - method_raw_out = self.testmethod.create_output( - dataset_name="RawTestOut", - dataset_idx=1) - - # Add a step - self.my_pipeline.steps.create( - transformation=self.testmethod, - step_num=1) - - # Add raw outmap - self.my_pipeline.create_raw_outcable( - raw_output_name="raw_out", - raw_output_idx=1, - source_step=1, - source=method_raw_out) - - self.assertEquals(self.my_pipeline.outputs.all().count(), 0) - self.my_pipeline.create_outputs() - self.assertEquals(self.my_pipeline.outputs.all().count(), 1) - - raw_output = self.my_pipeline.outputs.get(dataset_idx=1) - - self.assertEquals(raw_output.dataset_name, "raw_out") - - # Add another raw outmap - self.my_pipeline.create_raw_outcable( - raw_output_name="raw_out_2", - raw_output_idx=2, - source_step=1, - source=method_raw_out) - - self.my_pipeline.create_outputs() - self.assertEquals(self.my_pipeline.outputs.all().count(), 2) - - raw_output_2 = self.my_pipeline.outputs.get(dataset_idx=2) - - self.assertEquals(raw_output_2.dataset_name, "raw_out_2") - - -class PipelineStepInputCable_tests(PipelineTestCase): - - def test_PSIC_clean_and_completely_wired_CDT_equal_no_wiring_good(self): - # Define pipeline with mix_triplet_cdt (string, DNA, string) pipeline input - myPipeline = self.test_PF.members.create(revision_name="foo", revision_desc="Foo version", user=self.user) - myPipeline_input = myPipeline.create_input( - compounddatatype=self.mix_triplet_cdt, - dataset_name="pipe_in", dataset_idx=1 - ) - - # Define method with doublet_cdt input (string, string), add it to the pipeline, and cable it - m = Method(revision_name="s4", revision_desc="s4", family=self.test_MF, driver=self.script_4_1_CRR, - user=self.user) - m.save() - method_input = m.create_input(compounddatatype=self.mix_triplet_cdt, dataset_name="method_in", dataset_idx=1) - pipelineStep = myPipeline.steps.create(transformation=m, step_num=1) - pipeline_cable = pipelineStep.cables_in.create(dest=method_input, source_step=0, source=myPipeline_input) - - self.assertEquals(pipeline_cable.clean(), None) - self.assertEquals(pipeline_cable.clean_and_completely_wired(), None) - self.assertEquals(pipelineStep.clean(), None) - self.assertEquals(pipelineStep.complete_clean(), None) - - def test_PSIC_clean_and_completely_wired_CDT_not_equal_wires_exist_shuffled_wiring_good(self): - # Wire from a triplet into a double: - # A -> z - # B -> NULL (Not necessary) - # C -> x - - # Define pipeline with mix_triplet_cdt (string, DNA, string) pipeline input - myPipeline = self.test_PF.members.create(revision_name="foo", revision_desc="Foo version", user=self.user) - myPipeline_input = myPipeline.create_input( - compounddatatype=self.mix_triplet_cdt, - dataset_name="pipe_in", - dataset_idx=1 - ) - - # Define method with doublet_cdt input (string, string), add it to the pipeline, and cable it - m = Method(revision_name="s4", revision_desc="s4", family=self.test_MF, driver=self.script_4_1_CRR, - user=self.user) - m.save() - method_input = m.create_input(compounddatatype=self.doublet_cdt, dataset_name="method_in", dataset_idx=1) - pipelineStep = myPipeline.steps.create(transformation=m, step_num=1) - pipeline_cable = pipelineStep.cables_in.create(dest=method_input, source_step=0, source=myPipeline_input) - - # wire1 = string->string - wire1 = pipeline_cable.custom_wires.create( - source_pin=myPipeline_input.get_cdt().members.get(column_idx=3), - dest_pin=method_input.get_cdt().members.get(column_idx=2)) - - # The cable is clean but not complete - errorMessage = "Destination member .* has no wires leading to it" - self.assertEquals(pipeline_cable.clean(), None) - self.assertRaisesRegexp(ValidationError, errorMessage, pipeline_cable.clean_and_completely_wired) - - # wire2 = DNA->string - wire2 = pipeline_cable.custom_wires.create(source_pin=myPipeline_input.get_cdt().members.get(column_idx=2), - dest_pin=method_input.get_cdt().members.get(column_idx=1)) - - self.assertEquals(wire1.clean(), None) - self.assertEquals(wire2.clean(), None) - self.assertEquals(pipeline_cable.clean(), None) - self.assertEquals(pipeline_cable.clean_and_completely_wired(), None) - self.assertEquals(pipelineStep.clean(), None) - self.assertEquals(pipelineStep.complete_clean(), None) - - def test_PSIC_clean_and_completely_wired_CDT_not_equal_wires_exist_compatible_wiring_good(self): - # A -> x - # A -> y - - # Define pipeline with mix_triplet_cdt (string, DNA, string) pipeline input - myPipeline = self.test_PF.members.create(revision_name="foo", revision_desc="Foo version", user=self.user) - myPipeline_input = myPipeline.create_input(compounddatatype=self.mix_triplet_cdt, dataset_name="pipe_in", - dataset_idx=1) - - # Define method with doublet_cdt input (string, string), add it to the pipeline, and cable it - m = Method(revision_name="s4", revision_desc="s4", family=self.test_MF, driver=self.script_4_1_CRR, - user=self.user) - m.save() - method_input = m.create_input(compounddatatype=self.doublet_cdt, dataset_name="method_in", dataset_idx=1) - pipelineStep = myPipeline.steps.create(transformation=m, step_num=1) - pipeline_cable = pipelineStep.cables_in.create(dest=method_input, source_step=0, source=myPipeline_input) - - # wire1 = string->string - wire1 = pipeline_cable.custom_wires.create(source_pin=myPipeline_input.get_cdt().members.get(column_idx=2), - dest_pin=method_input.get_cdt().members.get(column_idx=2)) - - # wire2 = DNA->string - wire2 = pipeline_cable.custom_wires.create(source_pin=myPipeline_input.get_cdt().members.get(column_idx=2), - dest_pin=method_input.get_cdt().members.get(column_idx=1)) - - self.assertEquals(wire1.clean(), None) - self.assertEquals(wire2.clean(), None) - self.assertEquals(pipeline_cable.clean(), None) - self.assertEquals(pipelineStep.clean(), None) - self.assertEquals(pipelineStep.complete_clean(), None) - - def test_PSIC_clean_and_completely_wired_not_quenched(self): - # x -> x - # NULL -> y - # z -> z - - # Define pipeline with mix_triplet_cdt (string, DNA, string) pipeline input - myPipeline = self.test_PF.members.create(revision_name="foo", revision_desc="Foo version", user=self.user) - myPipeline_input = myPipeline.create_input( - compounddatatype=self.mix_triplet_cdt, - dataset_name="pipe_in", - dataset_idx=1 - ) - - # Define method with triplet_cdt input (string, string, string), add it to the pipeline, and cable it - m = Method(revision_name="s4", revision_desc="s4", family=self.test_MF, driver=self.script_4_1_CRR, - user=self.user) - m.save() - method_input = m.create_input(compounddatatype=self.triplet_cdt, dataset_name="method_in", dataset_idx=1) - pipelineStep = myPipeline.steps.create(transformation=m, step_num=1) - pipeline_cable = pipelineStep.cables_in.create(dest=method_input, source_step=0, source=myPipeline_input) - - # wire1 = string->string - wire1 = pipeline_cable.custom_wires.create( - source_pin=myPipeline_input.get_cdt().members.get(column_idx=1), - dest_pin=method_input.get_cdt().members.get(column_idx=1)) - - wire3 = pipeline_cable.custom_wires.create( - source_pin=myPipeline_input.get_cdt().members.get(column_idx=3), - dest_pin=method_input.get_cdt().members.get(column_idx=3)) - - self.assertEquals(wire1.clean(), None) - self.assertEquals(wire3.clean(), None) - self.assertEquals(pipeline_cable.clean(), None) - - # FIXME: Should pipelineStep.clean invoke pipeline_cable.clean_and_completely_quenched() ? - errorMessage = re.escape('Destination member "b: string" has no wires leading to it') - self.assertRaisesRegexp(ValidationError, errorMessage, pipeline_cable.clean_and_completely_wired) - self.assertRaisesRegexp(ValidationError, errorMessage, pipelineStep.clean) - self.assertRaisesRegexp(ValidationError, errorMessage, pipelineStep.complete_clean) - self.assertRaisesRegexp(ValidationError, errorMessage, myPipeline.complete_clean) - - def _make_log(self, pipeline, output_file, source): - """ - Helper function to make an ExecLog and RSIC for a pipeline. - """ - run = pipeline.pipeline_instances.create(user=self.user) - pipelinestep = self.DNAcompv1_p.steps.first() - runstep = pipelinestep.pipelinestep_instances.create(run=run) - psic = pipelinestep.cables_in.first() - rsic = psic.psic_instances.create(dest_runstep=runstep) - log = ExecLog(record=rsic, invoking_record=rsic) - log.save() - psic.run_cable(source, output_file, rsic, log) - return log, rsic - - def _setup_dirs(self): - """ - Helper function to make a temp directory and output file. - """ - scratch_dir = tempfile.mkdtemp() - output_file = os.path.join(scratch_dir, "output") - return scratch_dir, output_file - - def _log_checks(self, log, rsic): - """ - Helper function to check that an ExecLog made from an RSIC is coherent. - """ - self.assertEqual(log.record, rsic) - self.assertEqual(log.start_time.date(), timezone.now().date()) - self.assertEqual(log.end_time.date(), timezone.now().date()) - self.assertEqual(log.start_time < timezone.now(), True) - self.assertEqual(log.end_time < timezone.now(), True) - self.assertEqual(log.start_time <= log.end_time, True) - self.assertEqual(log.is_complete(), True) - self.assertEqual(log.complete_clean(), None) - self.assertEqual(len(log.missing_outputs()), 0) - self.assertEqual(log.is_successful(), True) - - def test_execlog_psic_run_cable_file(self): - """ - Check the coherence of an ExecLog created by running a cable with a Dataset. - """ - scratch_dir, output_file = self._setup_dirs() - temporary_file, safe_fn = tempfile.mkstemp(dir=self.workdir) - os.close(temporary_file) - datafile = open(safe_fn, "w") - datafile.write(",".join([m.column_name for m in self.DNAinput_cdt.members.all()])) - datafile.write("\n") - datafile.write("ATCG\n") - datafile.close() - - log, rsic = self._make_log(self.DNAcompv1_p, output_file, datafile.name) - self._log_checks(log, rsic) - shutil.rmtree(scratch_dir) - - -# August 29, 2013: reworked to handle new design for outcables. -class CustomOutputWiringTests(PipelineTestCase): - - def test_CustomOutputCableWire_clean_references_invalid_CDTM(self): - self.my_pipeline = self.test_PF.members.create(revision_name="foo", revision_desc="Foo version", - user=self.user) - self.my_pipeline.create_input(compounddatatype=self.triplet_cdt, dataset_name="pipeline_in_1", - dataset_idx=1) - - # Give the method self.triplet_cdt output - method_out = self.testmethod.create_output(dataset_name="TestOut", dataset_idx=1, - compounddatatype=self.triplet_cdt) - - # Add a step - self.my_pipeline.steps.create(transformation=self.testmethod, step_num=1) - - # Add an output cable - outcable1 = self.my_pipeline.create_outcable(output_name="blah", output_idx=1, source_step=1, source=method_out) - - # Add custom wiring from an irrelevent CDTM - badwire = outcable1.custom_wires.create(source_pin=self.doublet_cdt.members.first(), - dest_pin=self.triplet_cdt.members.first()) - - errorMessage = re.escape('Source pin "x: string" does not come from compounddatatype ' - '"(a: string, b: string, c: string)"') - - self.assertRaisesRegexp(ValidationError, errorMessage, badwire.clean) - self.assertRaisesRegexp(ValidationError, errorMessage, outcable1.clean) - self.assertRaisesRegexp(ValidationError, errorMessage, self.my_pipeline.clean) - - def test_Pipeline_create_outputs_for_creation_of_output_CDT(self): - self.my_pipeline = self.test_PF.members.create(revision_name="foo", revision_desc="Foo version", - user=self.user) - - self.my_pipeline.create_input( - compounddatatype=self.triplet_cdt, - dataset_name="pipeline_in_1", - dataset_idx=1) - - # Give the method self.triplet_cdt output - method_out = self.testmethod.create_output( - dataset_name="TestOut", - dataset_idx=1, - compounddatatype=self.mix_triplet_cdt) - - # Add a step - self.my_pipeline.steps.create( - transformation=self.testmethod, step_num=1) - - # Add an output cable with the following output CDT: - # column 1: "col1_str", type string_dt (from 1st col of triplet) - # column 2: "col2_DNA", type DNA_dt (from 2nd col of triplet) - # column 3: "col3_str", type string_dt (from 1st col of triplet) - # column 4: "col4_str", type string_dt (from 3rd col of triplet) - new_cdt = CompoundDatatype(user=self.user) - new_cdt.save() - pin1 = new_cdt.members.create(column_name="col1_str", column_idx=1, - datatype=self.string_dt) - pin2 = new_cdt.members.create(column_name="col2_DNA", column_idx=2, - datatype=self.DNA_dt) - pin3 = new_cdt.members.create(column_name="col3_str", column_idx=3, - datatype=self.string_dt) - pin4 = new_cdt.members.create(column_name="col4_str", column_idx=4, - datatype=self.string_dt) - - outcable1 = self.my_pipeline.outcables.create( - output_name="blah", - output_idx=1, - source_step=1, - source=method_out, - output_cdt=new_cdt) - - # Add wiring - outcable1.custom_wires.create( - source_pin=method_out.get_cdt().members.all()[0], - dest_pin=pin1) - - outcable1.custom_wires.create( - source_pin=method_out.get_cdt().members.all()[1], - dest_pin=pin2) - - outcable1.custom_wires.create( - source_pin=method_out.get_cdt().members.all()[0], - dest_pin=pin3) - - outcable1.custom_wires.create( - source_pin=method_out.get_cdt().members.all()[2], - dest_pin=pin4) - - self.assertEquals(self.my_pipeline.outputs.all().count(), 0) - self.my_pipeline.create_outputs() - self.assertEquals(self.my_pipeline.outputs.all().count(), 1) - - pipeline_out_members = self.my_pipeline.outputs.all()[0].get_cdt().members.all() - - self.assertEquals(pipeline_out_members.count(), 4) - - member = pipeline_out_members.get(column_idx=1) - self.assertEquals(member.column_name, "col{}_str".format(1)) - self.assertEquals(member.datatype, self.string_dt) - - member = pipeline_out_members.get(column_idx=2) - self.assertEquals(member.column_name, "col{}_DNA".format(2)) - self.assertEquals(member.datatype, self.DNA_dt) - - member = pipeline_out_members.get(column_idx=3) - self.assertEquals(member.column_name, "col{}_str".format(3)) - self.assertEquals(member.datatype, self.string_dt) - - member = pipeline_out_members.get(column_idx=4) - self.assertEquals(member.column_name, "col{}_str".format(4)) - self.assertEquals(member.datatype, self.string_dt) - - -def create_pipeline_deserialization_environment(case): - """ - Set up stuff that will help with testing Pipeline deserialization. - - The "sandbox" testing environment must be set up already, either - by directly calling create_sandbox_testing_tools_environment or - by including a fixture that had called it. - """ - case.kive_user = kive_user() - case.everyone_group = everyone_group() - - # Explicitly load objects that are defined in create_sandbox_testing... - # in case we are using a fixture. - case.user_bob = User.objects.get(username="bob") - case.coderesource_noop = CodeResource.objects.get( - user=case.user_bob, - name="noop" - ) - case.coderev_noop = case.coderesource_noop.revisions.get(revision_name="1") - case.noop_mf = MethodFamily.objects.get(name="string noop") - case.method_noop = case.noop_mf.members.get(revision_number=1) - case.noop_raw_mf = MethodFamily.objects.get(name="raw noop", user=case.user_bob) - case.method_noop_raw = case.noop_raw_mf.members.get(revision_number=1) - - # Retrieve the CDT defined in create_sandbox_testing_tools_environment - # called "self.cdt_string", or an equivalent. - bob_string_dt = Datatype.objects.get( - user=case.user_bob, - name="my_string", - description="sequences of ASCII characters" - ) - possible_cdt_string_members = CompoundDatatypeMember.objects.filter( - column_name="word", - column_idx=1, - datatype=bob_string_dt - ) - possible_cdt_strings = [x.compounddatatype for x in possible_cdt_string_members] - case.cdt_string = possible_cdt_strings[0] - - case.duck_context = DuckContext() - - case.test_pf = PipelineFamily( - user=case.kive_user, - name="test", - description="Test family" - ) - case.test_pf.save() - case.test_pf.groups_allowed.add(case.everyone_group) - - # Set up a CDT with two elements to allow some wiring to occur. - case.STR = Datatype.objects.get(pk=datatypes.STR_PK) - - # A CDT composed of two builtin-STR columns. - case.string_doublet = CompoundDatatype(user=case.user_bob) - case.string_doublet.save() - case.string_doublet.members.create(datatype=case.STR, column_name="column1", column_idx=1) - case.string_doublet.members.create(datatype=case.STR, column_name="column2", column_idx=2) - case.string_doublet.grant_everyone_access() - - # A CDT composed of one builtin-STR column. - case.string_singlet = CompoundDatatype(user=case.user_bob) - case.string_singlet.save() - case.string_singlet.members.create(datatype=case.STR, column_name="col1", column_idx=1) - case.string_singlet.grant_everyone_access() - - # Here is a dictionary that can be deserialized into a Pipeline. - case.noop_input_name = case.method_noop.inputs.first().dataset_name - case.noop_output_name = case.method_noop.outputs.first().dataset_name - case.pipeline_dict = { - "family": "test", - "revision_name": "v1", - "revision_desc": "first version", - "revision_parent": None, - - "user": case.kive_user.username, - "users_allowed": [], - "groups_allowed": [case.everyone_group.name], - - "inputs": [ - { - "dataset_name": "input_to_not_touch", - "dataset_idx": 1, - "x": 0.05, - "y": 0.5, - "structure": { - "compounddatatype": case.cdt_string.pk, - "min_row": None, - "max_row": None - } - } - ], - "steps": [ - { - "transformation": case.method_noop.pk, - "step_num": 1, - "x": 0.2, - "y": 0.5, - "name": "step 1", - "cables_in": [ - { - # The pipeline input doesn't exist yet so we have to specify - # it by name. - "source_dataset_name": "input_to_not_touch", - "source_step": 0, - "dest_dataset_name": case.noop_input_name, - "custom_wires": [], - "keep_output": False - } - ], - "new_outputs_to_delete_names": [] - }, - { - "transformation": case.method_noop.pk, - "step_num": 2, - "x": 0.4, - "y": 0.5, - "name": "step 2", - "cables_in": [ - { - # Here we can specify source directly. - "source_dataset_name": case.noop_output_name, - "source_step": 1, - "dest_dataset_name": case.noop_input_name, - "custom_wires": [], - "keep_output": False - } - ], - }, - { - "transformation": case.method_noop.pk, - "step_num": 3, - "x": 0.6, - "y": 0.5, - "name": "step 3", - "cables_in": [ - { - "source_dataset_name": case.noop_output_name, - "source_step": 2, - "dest_dataset_name": case.noop_input_name, - "custom_wires": [], - "keep_output": False - } - ], - "new_outputs_to_delete_names": [] - } - ], - "outcables": [ - { - "output_idx": 1, - "output_name": "untouched_output", - "output_cdt": case.cdt_string.pk, - "source_step": 3, - "source_dataset_name": case.noop_output_name, - "x": 0.85, - "y": 0.5, - "custom_wires": [] - } - ] - } - - case.method_doublet_noop = tools.make_first_method( - "string doublet noop", - "a noop on a two-column input", - case.coderev_noop, - case.user_bob) - case.method_doublet_noop.grant_everyone_access() - case.doublet_input_name = "doublets" - case.doublet_output_name = "untouched_doublets" - tools.simple_method_io( - case.method_doublet_noop, - case.string_doublet, - case.doublet_input_name, - case.doublet_output_name - ) - - # This defines a pipeline with custom wiring. - case.pipeline_cw_dict = { - "family": "test", - "revision_name": "v2_c2", - "revision_desc": "Custom wiring tester", - "revision_parent": None, - - "user": case.kive_user.username, - "users_allowed": [case.kive_user.username], - "groups_allowed": [], - - "inputs": [ - { - "dataset_name": "input_to_not_touch", - "dataset_idx": 1, - "x": 0.05, - "y": 0.5, - "structure": { - "compounddatatype": case.cdt_string.pk, - "min_row": None, - "max_row": None - } - } - ], - "steps": [ - { - "transformation": case.method_doublet_noop.pk, - "step_num": 1, - "x": 0.2, - "y": 0.5, - "name": "step 1", - "cables_in": [ - { - # The pipeline input doesn't exist yet so we have to specify - # it by name. - "source_dataset_name": "input_to_not_touch", - "source_step": 0, - "dest_dataset_name": case.doublet_input_name, - "custom_wires": [ - { - "source_pin": case.cdt_string.members.first().pk, - "dest_pin": case.string_doublet.members.get(column_idx=1).pk - }, - { - "source_pin": case.cdt_string.members.first().pk, - "dest_pin": case.string_doublet.members.get(column_idx=2).pk - }, - ], - "keep_output": False - } - ], - }, - { - "transformation": case.method_noop.pk, - "step_num": 2, - "x": 0.4, - "y": 0.5, - "name": "step 2", - "cables_in": [ - { - # Here we can specify source directly. - "source_dataset_name": case.doublet_output_name, - "source_step": 1, - "dest_dataset_name": case.noop_input_name, - "custom_wires": [ - { - "source_pin": case.string_doublet.members.get(column_idx=1).pk, - "dest_pin": case.cdt_string.members.first().pk - } - ], - "keep_output": False - } - ], - } - ], - "outcables": [ - { - "output_idx": 1, - "output_name": "untouched_output", - "output_cdt": case.string_doublet.pk, - "source_step": 2, - "source_dataset_name": case.noop_output_name, - "x": 0.85, - "y": 0.5, - "custom_wires": [ - { - "source_pin": case.cdt_string.members.first().pk, - "dest_pin": case.string_doublet.members.get(column_idx=1).pk - }, - { - "source_pin": case.cdt_string.members.first().pk, - "dest_pin": case.string_doublet.members.get(column_idx=2).pk - }, - ] - } - ] - } - - # This defines a pipeline that handles raw data. - case.raw_input_name = case.method_noop_raw.inputs.first().dataset_name - case.raw_output_name = case.method_noop_raw.outputs.first().dataset_name - case.pipeline_raw_dict = { - "family": "test", - "revision_name": "v3_raw", - "revision_desc": "Raw input tester", - "revision_parent": None, - - "user": case.kive_user.username, - "users_allowed": [case.kive_user.username], - "groups_allowed": [case.everyone_group], - - "inputs": [ - { - "dataset_name": "input_to_not_touch", - "dataset_idx": 1, - "x": 0.05, - "y": 0.5 - } - ], - "steps": [ - { - "transformation": case.method_noop_raw.pk, - "step_num": 1, - "x": 0.2, - "y": 0.5, - "name": "step 1", - "cables_in": [ - { - # The pipeline input doesn't exist yet so we have to specify - # it by name. - "source_dataset_name": "input_to_not_touch", - "source_step": 0, - "dest_dataset_name": case.raw_input_name, - "keep_output": False - } - ], - }, - { - "transformation": case.method_noop_raw.pk, - "step_num": 2, - "x": 0.4, - "y": 0.5, - "name": "step 2", - "cables_in": [ - { - # Here we can specify source directly. - "source_dataset_name": case.raw_output_name, - "source_step": 1, - "dest_dataset_name": case.raw_input_name, - "keep_output": False - } - ], - } - ], - "outcables": [ - { - "output_idx": 1, - "output_name": "untouched_output", - "source_step": 2, - "source_dataset_name": case.raw_output_name, - "x": 0.85, - "y": 0.5 - } - ] - } - - -@skipIfDBFeature('is_mocked') -class PipelineSerializerTests(TestCase): - """ - Tests of PipelineSerializer and its offshoots. - """ - def setUp(self): - tools.create_sandbox_testing_tools_environment(self) - create_pipeline_deserialization_environment(self) - - def tearDown(self): - tools.clean_up_all_files() - - def test_validate(self): - ps = PipelineSerializer(data=self.pipeline_dict, context=self.duck_context) - ps.is_valid() - self.assertTrue(ps.is_valid()) - - def test_validate_otd_good_name(self): - """ - Validating a properly-named output_to_delete. - """ - self.pipeline_dict["steps"][0]["new_outputs_to_delete_names"] = [self.noop_output_name] - ps = PipelineSerializer(data=self.pipeline_dict, context=self.duck_context) - self.assertTrue(ps.is_valid()) - - def test_validate_otd_bad_name(self): - """ - A step with a badly-named output to delete should fail. - """ - incorrect_name = "foo" - self.pipeline_dict["steps"][0]["new_outputs_to_delete_names"] = [incorrect_name] - ps = PipelineSerializer(data=self.pipeline_dict, context=self.duck_context) - self.assertFalse(ps.is_valid()) - self.assertEquals( - ps.errors["steps"][0]["non_field_errors"][0], - 'Step {} has no output named "{}"'.format(1, incorrect_name) - ) - - def test_validate_dest_bad_name(self): - """ - A PSIC with a destination that has a bad name should fail. - """ - incorrect_name = "foo" - self.pipeline_dict["steps"][0]["cables_in"][0]["dest_dataset_name"] = incorrect_name - ps = PipelineSerializer(data=self.pipeline_dict, context=self.duck_context) - self.assertFalse(ps.is_valid()) - self.assertEquals( - ps.errors["steps"][0]["non_field_errors"][0], - 'Step {} has no input named "{}"'.format(1, incorrect_name) - ) - - def test_validate_pipeline_input_source_bad_name(self): - """ - A PSIC with a Pipeline input source that has a bad name should fail. - """ - incorrect_name = "foo" - self.pipeline_dict["steps"][0]["cables_in"][0]["source_dataset_name"] = incorrect_name - ps = PipelineSerializer(data=self.pipeline_dict, context=self.duck_context) - self.assertFalse(ps.is_valid()) - self.assertEquals( - ps.errors["non_field_errors"][0], - 'Cable input with name "{}" does not exist'.format(incorrect_name) - ) - - def test_validate_step_output_source_bad_name(self): - """ - A PSIC with a step output source that has a bad name should fail. - """ - incorrect_name = "foo" - self.pipeline_dict["steps"][1]["cables_in"][0]["source_dataset_name"] = incorrect_name - ps = PipelineSerializer(data=self.pipeline_dict, context=self.duck_context) - self.assertFalse(ps.is_valid()) - self.assertEquals( - ps.errors["non_field_errors"][0], - 'Step {} has no output named "{}"'.format(1, incorrect_name) - ) - - def test_validate_step_output_source_bad_source_step(self): - """ - A PSIC with a step output source that has a bad step number should fail. - """ - bad_step_num = 100 - self.pipeline_dict["steps"][1]["cables_in"][0]["source_step"] = bad_step_num - ps = PipelineSerializer(data=self.pipeline_dict, context=self.duck_context) - self.assertFalse(ps.is_valid()) - self.assertEquals( - ps.errors["non_field_errors"][0], - "Step {} does not exist".format(bad_step_num) - ) - - def test_validate_custom_wires(self): - """ - Test validation of a Pipeline containing custom wires. - """ - ps = PipelineSerializer(data=self.pipeline_cw_dict, context=self.duck_context) - ps.is_valid() - self.assertTrue(ps.is_valid()) - - def test_validate_raw_input(self): - """ - Test validation of a Pipeline that only consists of raw data. - """ - ps = PipelineSerializer(data=self.pipeline_raw_dict, context=self.duck_context) - self.assertTrue(ps.is_valid()) - - def test_create(self): - ps = PipelineSerializer(data=self.pipeline_dict, - context=self.duck_context) - ps.is_valid() - pl = ps.save() - - # Probe the Pipeline to see if things are defined correctly. - pl_input = pl.inputs.first() - self.assertEquals(pl_input.structure.compounddatatype, self.cdt_string) - - step_1 = pl.steps.get(step_num=1) - step_2 = pl.steps.get(step_num=2) - step_3 = pl.steps.get(step_num=3) - - self.assertEquals(step_1.transformation.definite, self.method_noop) - - self.assertEquals(step_1.cables_in.count(), 1) - # Check that this cable got mapped correctly since we defined its source - # indirectly. - self.assertEquals(step_1.cables_in.first().source.definite, pl_input) - self.assertEquals(step_1.cables_in.first().custom_wires.count(), 0) - - self.assertEquals(step_2.cables_in.count(), 1) - self.assertEquals(step_2.cables_in.first().source.definite, self.method_noop.outputs.first()) - - self.assertEquals(step_3.cables_in.first().source_step, 2) - - outcable = pl.outcables.first() - self.assertEquals(pl.outcables.count(), 1) - self.assertEquals(outcable.custom_wires.count(), 0) - - output = pl.outputs.first() - self.assertEquals(pl.outputs.count(), 1) - self.assertEquals(output.dataset_name, "untouched_output") - self.assertEquals(output.dataset_idx, 1) - self.assertEquals(output.x, 0.85) - self.assertEquals(output.y, 0.5) - - def test_create_with_otd(self): - self.pipeline_dict["steps"][0]["new_outputs_to_delete_names"] = [self.noop_output_name] - ps = PipelineSerializer(data=self.pipeline_dict, - context=self.duck_context) - ps.is_valid() - pl = ps.save() - - # Probe the Pipeline to see if the output was properly registered for deletion. - step_1 = pl.steps.get(step_num=1) - self.assertEquals(step_1.outputs_to_delete.count(), 1) - self.assertEquals(step_1.outputs_to_delete.first(), step_1.transformation.outputs.first()) - - def test_create_with_custom_wires(self): - """ - Test that creation works when custom wires are used. - """ - ps = PipelineSerializer(data=self.pipeline_cw_dict, - context=self.duck_context) - ps.is_valid() - pl = ps.save() - - # Probe the Pipeline to see if things are defined correctly. - pl_input = pl.inputs.first() - self.assertEquals(pl_input.structure.compounddatatype, self.cdt_string) - - step_1 = pl.steps.get(step_num=1) - step_2 = pl.steps.get(step_num=2) - - self.assertEquals(step_1.transformation.definite, self.method_doublet_noop) - - self.assertEquals(step_1.cables_in.count(), 1) - self.assertEquals(step_1.cables_in.first().custom_wires.count(), 2) - wire_1 = step_1.cables_in.first().custom_wires.get(dest_pin__column_idx=1) - wire_2 = step_1.cables_in.first().custom_wires.get(dest_pin__column_idx=2) - self.assertEquals(wire_1.source_pin, self.cdt_string.members.first()) - self.assertEquals(wire_1.dest_pin, self.string_doublet.members.get(column_idx=1)) - self.assertEquals(wire_2.source_pin, self.cdt_string.members.first()) - self.assertEquals(wire_2.dest_pin, self.string_doublet.members.get(column_idx=2)) - - # Now check the wire defined on the input cable of step 2. - self.assertEquals(step_2.cables_in.first().custom_wires.count(), 1) - step_2_wire = step_2.cables_in.first().custom_wires.first() - self.assertEquals(step_2_wire.source_pin, self.string_doublet.members.get(column_idx=1)) - self.assertEquals(step_2_wire.dest_pin, self.cdt_string.members.first()) - - outcable = pl.outcables.first() - self.assertEquals(outcable.custom_wires.count(), 2) - out_wire_1 = outcable.custom_wires.get(dest_pin__column_idx=1) - out_wire_2 = outcable.custom_wires.get(dest_pin__column_idx=2) - self.assertEquals(out_wire_1.source_pin, self.cdt_string.members.first()) - self.assertEquals(out_wire_1.dest_pin, self.string_doublet.members.get(column_idx=1)) - self.assertEquals(out_wire_2.source_pin, self.cdt_string.members.first()) - self.assertEquals(out_wire_2.dest_pin, self.string_doublet.members.get(column_idx=2)) - - def test_create_raw_input(self): - """ - Test deserialization of a Pipeline hadnling raw data. - """ - raw_ps = PipelineSerializer(data=self.pipeline_raw_dict, - context=self.duck_context) - raw_ps.is_valid() - raw_pl = raw_ps.save() - - pl_input = raw_pl.inputs.first() - self.assertTrue(pl_input.is_raw()) - pl_output = raw_pl.outputs.first() - self.assertTrue(pl_output.is_raw()) - - def test_create_publish_on_submit(self): - """ - Testing publishing on submission. - """ - self.pipeline_dict["published"] = True - ps = PipelineSerializer(data=self.pipeline_dict, - context=self.duck_context) - self.assertTrue(ps.is_valid()) - pl = ps.save() - self.assertTrue(pl.published) - - -@skipIfDBFeature("is_mocked") -class PipelineFamilySerializerTests(TestCase): - """ - Tests of PipelineFamilySerializer and its offshoots. - """ - def setUp(self): - tools.create_sandbox_testing_tools_environment(self) - create_pipeline_deserialization_environment(self) - - # That created a PipelineFamily (self.test_pf). Create some Pipelines to go into it. - ps = PipelineSerializer(data=self.pipeline_dict, - context=self.duck_context) - ps.is_valid() - self.pl = ps.save() - - ps_raw = PipelineSerializer(data=self.pipeline_raw_dict, - context=self.duck_context) - ps_raw.is_valid() - self.pl_raw = ps_raw.save() - - ps_cw = PipelineSerializer(data=self.pipeline_cw_dict, - context=self.duck_context) - ps_cw.is_valid() - self.pl_cw = ps_cw.save() - - def tearDown(self): - tools.clean_up_all_files() - - def test_show_all_none_published(self): - """ - Test that the serializer properly filters out unpublished members. - """ - self.duck_context["only_is_published"] = False - - pfs = PipelineFamilySerializer(self.test_pf, context=self.duck_context) - test_pf_serialized = pfs.data - - # None of the pipelines are published. - self.assertEqual(len(test_pf_serialized["members"]), 3) - id_set = set(x["id"] for x in test_pf_serialized["members"]) - self.assertSetEqual(id_set, set([self.pl.pk, self.pl_raw.pk, self.pl_cw.pk])) - - def test_show_all_some_published(self): - """ - Test that the serializer properly filters out unpublished members. - """ - self.duck_context["only_is_published"] = False - - self.pl.published = True - - pfs = PipelineFamilySerializer(self.test_pf, context=self.duck_context) - test_pf_serialized = pfs.data - - # One of the pipelines are published. - self.assertEqual(len(test_pf_serialized["members"]), 3) - id_set = set(x["id"] for x in test_pf_serialized["members"]) - self.assertSetEqual(id_set, set([self.pl.pk, self.pl_raw.pk, self.pl_cw.pk])) - - def test_show_all_all_published(self): - """ - Test that the serializer properly filters out unpublished members. - """ - self.duck_context["only_is_published"] = False - - self.pl.published = True - self.pl_raw.published = True - self.pl_cw.published = True - - pfs = PipelineFamilySerializer(self.test_pf, context=self.duck_context) - test_pf_serialized = pfs.data - - # One of the pipelines are published. - self.assertEqual(len(test_pf_serialized["members"]), 3) - id_set = set(x["id"] for x in test_pf_serialized["members"]) - self.assertSetEqual(id_set, set([self.pl.pk, self.pl_raw.pk, self.pl_cw.pk])) - - def test_only_is_published_none_published(self): - """ - Test that the serializer properly filters out unpublished members. - """ - self.duck_context["only_is_published"] = True - - pfs = PipelineFamilySerializer(self.test_pf, context=self.duck_context) - test_pf_serialized = pfs.data - - # None of the pipelines are published. - self.assertEqual(len(test_pf_serialized["members"]), 0) - - def test_only_is_published_some_published(self): - """ - Test that the serializer properly filters out unpublished members. - """ - self.duck_context["only_is_published"] = True - - self.pl.published = True - self.pl.save() - - pfs = PipelineFamilySerializer(self.test_pf, context=self.duck_context) - test_pf_serialized = pfs.data - - # One of the pipelines are published. - self.assertEqual(len(test_pf_serialized["members"]), 1) - self.assertEqual(test_pf_serialized["members"][0]["id"], self.pl.pk) - - def test_only_is_published_all_published(self): - """ - Test that the serializer properly filters out unpublished members. - """ - self.duck_context["only_is_published"] = True - - self.pl.published = True - self.pl.save() - self.pl_raw.published = True - self.pl_raw.save() - self.pl_cw.published = True - self.pl_cw.save() - - pfs = PipelineFamilySerializer(self.test_pf, context=self.duck_context) - test_pf_serialized = pfs.data - - # One of the pipelines are published. - self.assertEqual(len(test_pf_serialized["members"]), 3) - id_set = set(x["id"] for x in test_pf_serialized["members"]) - self.assertSetEqual(id_set, set([self.pl.pk, self.pl_raw.pk, self.pl_cw.pk])) - - -@skipIfDBFeature('is_mocked') -class PipelineApiTests(BaseTestCases.ApiTestCase): - fixtures = ['simple_run'] - - def setUp(self): - super(PipelineApiTests, self).setUp() - - self.list_path = reverse("pipeline-list") - self.detail_pk = 5 - self.detail_path = reverse("pipeline-detail", - kwargs={'pk': self.detail_pk}) - - self.list_view, _, _ = resolve(self.list_path) - self.detail_view, _, _ = resolve(self.detail_path) - - def test_list(self): - request = self.factory.get(self.list_path) - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - expected_count = Pipeline.objects.count() - self.assertEquals(len(response.data), expected_count) - self.assertEquals(response.data[0]['family'], 'P_basic') - self.assertEquals(response.data[0]['revision_name'], 'v1') - self.assertEquals(response.data[0]['inputs'][0]['dataset_name'], 'basic_in') - - def test_detail(self): - request = self.factory.get(self.detail_path) - force_authenticate(request, user=self.kive_user) - response = self.detail_view(request, pk=self.detail_pk) - self.assertEquals(response.data['revision_name'], 'pE_name') - self.assertEquals(response.data['inputs'][0]['dataset_name'], 'E1_in') - - def test_removal_plan(self): - removal_path = reverse("pipeline-removal-plan", - kwargs={'pk': self.detail_pk}) - removal_view, _, _ = resolve(removal_path) - request = self.factory.get(removal_path) - force_authenticate(request, user=self.kive_user) - - response = removal_view(request, pk=self.detail_pk) - - self.assertEquals(response.data['Pipelines'], 1) - - def test_removal(self): - start_count = Pipeline.objects.count() - request = self.factory.delete(self.detail_path) - force_authenticate(request, user=self.kive_user) - response = self.detail_view(request, pk=self.detail_pk) - self.assertEquals(response.status_code, status.HTTP_204_NO_CONTENT) - - end_count = Pipeline.objects.count() - self.assertEquals(end_count, start_count - 1) - - def test_step_updates(self): - step_updates_path = reverse("pipeline-step-updates", - kwargs={'pk': self.detail_pk}) - step_updates_view, _, _ = resolve(step_updates_path) - request = self.factory.get(step_updates_path) - force_authenticate(request, user=self.kive_user) - - response = step_updates_view(request, pk=self.detail_pk) - - update = response.data[0] - self.assertEqual(update['step_num'], 1) - - def test_create(self): - # Note that the "sandbox" testing environment has already been set - # up in the "simple_run" fixture. - create_pipeline_deserialization_environment(self) - request = self.factory.post(self.list_path, self.pipeline_dict, format="json") - force_authenticate(request, user=self.kive_user) - response = self.list_view(request) - - if response.exception: - self.fail(response.data) - # Probe the new object. - new_pipeline = self.test_pf.members.get(revision_name=self.pipeline_dict["revision_name"]) - self.assertEquals(new_pipeline.steps.count(), 3) - self.assertEquals(new_pipeline.outcables.count(), 1) - self.assertEquals(new_pipeline.outcables.first().output_name, "untouched_output") - - def create_new_code_revision(self, coderesource): - contents = "print('This is the new code.')" - with tempfile.TemporaryFile() as f: - f.write(contents) - revision = CodeResourceRevision( - coderesource=coderesource, - revision_name="new", - revision_desc="just print a message", - content_file=File(f), - user=self.user_bob) - revision.clean() - revision.save() - return revision - - def test_create_with_new_method(self): - create_pipeline_deserialization_environment(self) - revision = self.create_new_code_revision(self.coderesource_noop) - - step_dict = self.pipeline_dict['steps'][0] - step_dict['new_code_resource_revision_id'] = revision.id - request = self.factory.post(self.list_path, - self.pipeline_dict, - format="json") - force_authenticate(request, user=self.kive_user) - response = self.list_view(request) - - if response.exception: - self.fail(response.data) - # Probe the new object. - new_pipeline = self.test_pf.members.get( - revision_name=self.pipeline_dict["revision_name"]) - method = revision.methods.first() - self.assertIsNotNone(method, 'method expected for new code revision') - step = new_pipeline.steps.get(step_num=1) - self.assertEqual(step.transformation.display_name, method.display_name) - - def test_create_with_new_method_for_dependency(self): - """ - Create a new Pipeline revision to suit a newly-updated step dependency. - """ - create_pipeline_deserialization_environment(self) - - # Add a dependency to self.method_noop. - dependency_revision = CodeResourceRevision.objects.exclude( - coderesource=self.coderesource_noop).earliest('id') - new_dependency_revision = self.create_new_code_revision( - dependency_revision.coderesource) - self.method_noop.dependencies.create(requirement=dependency_revision) - - step_dict = self.pipeline_dict['steps'][0] - step_dict['new_dependency_ids'] = [new_dependency_revision.id] - request = self.factory.post(self.list_path, - self.pipeline_dict, - format="json") - force_authenticate(request, user=self.kive_user) - response = self.list_view(request) - - if response.exception: - self.fail(response.data) - # Probe the new object. - new_pipeline = self.test_pf.members.get( - revision_name=self.pipeline_dict["revision_name"]) - noop_revision_dependency = new_dependency_revision.used_by.first() - self.assertIsNotNone(noop_revision_dependency, 'noop_revision expected for new dependency') - method = noop_revision_dependency.method - self.assertIsNotNone(method, 'method expected for noop_revision') - step = new_pipeline.steps.get(step_num=1) - self.assertEqual(step.transformation.display_name, method.display_name) - - def test_create_calls_clean(self): - """ - Attempting to create a Pipeline should call complete_clean. - """ - # Note that the "sandbox" testing environment has already been set - # up in the "simple_run" fixture. - create_pipeline_deserialization_environment(self) - - self.pipeline_dict["steps"][0]["cables_in"] = [] - request = self.factory.post(self.list_path, self.pipeline_dict, format="json") - force_authenticate(request, user=self.kive_user) - # This should barf. - response = self.list_view(request) - - self.assertDictEqual( - response.data, - {'non_field_errors': 'Input "strings" to transformation at step 1 is not cabled'}) - - def test_partial_update_published(self): - """ - Test PATCHing a Pipeline to update its published status. - """ - # This is defined in simple_run. - basic_pf = PipelineFamily.objects.get(name="P_basic") - - version_to_publish = basic_pf.members.first() - patch_data = { - "published": "true" - } - - patch_path = reverse("pipeline-detail", - kwargs={"pk": version_to_publish.pk}) - request = self.factory.patch(patch_path, patch_data, format="json") - force_authenticate(request, user=self.kive_user) - response = self.detail_view(request, pk=version_to_publish.pk) - self.assertEquals(response.status_code, status.HTTP_200_OK) - - # Probe version_to_publish to check that it's been properly updated. - version_to_publish = Pipeline.objects.get(pk=version_to_publish.pk) - self.assertTrue(version_to_publish.published) - - # Now unpublish it. - patch_data["published"] = "false" - - request = self.factory.patch(patch_path, patch_data, format="json") - force_authenticate(request, user=self.kive_user) - response = self.detail_view(request, pk=version_to_publish.pk) - self.assertEquals(response.status_code, status.HTTP_200_OK) - version_to_publish = Pipeline.objects.get(pk=version_to_publish.pk) - self.assertFalse(version_to_publish.published) - - -@skipIfDBFeature('is_mocked') -class PipelineFamilyApiTests(BaseTestCases.ApiTestCase): - fixtures = ['simple_run'] - - def setUp(self): - super(PipelineFamilyApiTests, self).setUp() - - self.list_path = reverse("pipelinefamily-list") - self.detail_pk = 2 - self.detail_path = reverse("pipelinefamily-detail", - kwargs={'pk': self.detail_pk}) - self.removal_path = reverse("pipelinefamily-removal-plan", - kwargs={'pk': self.detail_pk}) - - self.list_view, _, _ = resolve(self.list_path) - self.detail_view, _, _ = resolve(self.detail_path) - self.removal_view, _, _ = resolve(self.removal_path) - - def test_list(self): - request = self.factory.get(self.list_path) - force_authenticate(request, user=self.kive_user) - response = self.list_view(request, pk=None) - - expected_count = PipelineFamily.objects.count() - self.assertEquals(len(response.data), expected_count) - self.assertEquals(response.data[1]['name'], 'Pipeline_family') - - pf = PipelineFamily.objects.get(name="Pipeline_family") - expected_revision_pks = [x.pk for x in pf.members.all()] - actual_revision_pks = [x['id'] for x in response.data[1]['members']] - self.assertItemsEqual(expected_revision_pks, actual_revision_pks) - - def test_detail(self): - request = self.factory.get(self.detail_path) - force_authenticate(request, user=self.kive_user) - response = self.detail_view(request, pk=self.detail_pk) - self.assertEquals(response.data['name'], 'P_basic') - - basic_family = PipelineFamily.objects.get(name="P_basic") - expected_revision_pks = [x.pk for x in basic_family.members.all()] - actual_revision_pks = [x['id'] for x in response.data['members']] - self.assertItemsEqual(expected_revision_pks, actual_revision_pks) - - def test_removal_plan(self): - request = self.factory.get(self.removal_path) - force_authenticate(request, user=self.kive_user) - response = self.removal_view(request, pk=self.detail_pk) - self.assertEquals(response.data['PipelineFamilies'], 1) - - def test_removal(self): - start_count = PipelineFamily.objects.count() - request = self.factory.delete(self.detail_path) - force_authenticate(request, user=self.kive_user) - response = self.detail_view(request, pk=self.detail_pk) - self.assertEquals(response.status_code, status.HTTP_204_NO_CONTENT) - - end_count = PipelineFamily.objects.count() - self.assertEquals(end_count, start_count - 1) - - def test_create(self): - pf_name = "Test PipelineFamily" - pf_description = "For testing the creation of a PipelineFamily through the API." - pf_data = { - "name": pf_name, - "description": pf_description, - "users_allowed": [], - "groups_allowed": [everyone_group().name] - } - - request = self.factory.post(self.list_path, pf_data, format="json") - force_authenticate(request, user=self.kive_user) - self.list_view(request) - - # Probe the resulting new PipelineFamily. - new_pf = PipelineFamily.objects.get(name=pf_name) - self.assertEquals(new_pf.description, pf_description) - self.assertEquals(new_pf.members.count(), 0) - - -@skipIfDBFeature("is_mocked") -class PipelineFamilyApiOnlyIsPublishedTests(BaseTestCases.ApiTestCase): - """ - Tests whether unpublished pipelines are properly filtered if the user is/is not a developer. - """ - def setUp(self): - super(PipelineFamilyApiOnlyIsPublishedTests, self).setUp() - - self.list_path = reverse("pipelinefamily-list") - self.detail_pk = 2 - self.detail_path = reverse("pipelinefamily-detail", - kwargs={'pk': self.detail_pk}) - self.removal_path = reverse("pipelinefamily-removal-plan", - kwargs={'pk': self.detail_pk}) - - self.list_view, _, _ = resolve(self.list_path) - self.detail_view, _, _ = resolve(self.detail_path) - self.removal_view, _, _ = resolve(self.removal_path) - - # Create a PipelineFamily to use in the tests. - tools.create_sandbox_testing_tools_environment(self) - create_pipeline_deserialization_environment(self) - - # That created a PipelineFamily (self.test_pf). Create some Pipelines to go into it. - # This one is published. - ps = PipelineSerializer(data=self.pipeline_dict, - context=self.duck_context) - ps.is_valid() - self.pl = ps.save() - self.pl.published = True - self.pl.save() - - # This one is unpublished. - ps_raw = PipelineSerializer(data=self.pipeline_raw_dict, - context=self.duck_context) - ps_raw.is_valid() - self.pl_raw = ps_raw.save() - - # This one is published. - ps_cw = PipelineSerializer(data=self.pipeline_cw_dict, - context=self.duck_context) - ps_cw.is_valid() - self.pl_cw = ps_cw.save() - self.pl_cw.published = True - self.pl_cw.save() - - def tearDown(self): - tools.clean_up_all_files() - - def test_pipelines_unfiltered_for_developer(self): - """ - All pipelines should show up if the user is a developer. - """ - dev = User.objects.create_user("dev", "dev@developers.net", "foobar") - dev.groups.add(everyone_group()) - dev.groups.add(Group.objects.get(name="Developers")) - - request = self.factory.get(self.detail_path) - force_authenticate(request, user=dev) - response = self.detail_view(request, pk=self.test_pf.pk) - - expected_revision_pks = [self.pl.pk, self.pl_cw.pk, self.pl_raw.pk] - actual_revision_pks = [x['id'] for x in response.data['members']] - self.assertItemsEqual(expected_revision_pks, actual_revision_pks) - - def test_pipelines_filtered_for_non_developer(self): - """ - Only published pipelines should show up if the user is not a developer. - """ - non_dev = User.objects.create_user("non_dev", "non_dev@users.net", "barf") - non_dev.groups.add(everyone_group()) - - request = self.factory.get(self.detail_path) - force_authenticate(request, user=non_dev) - response = self.detail_view(request, pk=self.test_pf.pk) - - expected_revision_pks = [self.pl.pk, self.pl_cw.pk] - actual_revision_pks = [x['id'] for x in response.data['members']] - self.assertItemsEqual(expected_revision_pks, actual_revision_pks) \ No newline at end of file diff --git a/kive/pipeline/tests_mock.py b/kive/pipeline/tests_mock.py deleted file mode 100644 index 8620dec26..000000000 --- a/kive/pipeline/tests_mock.py +++ /dev/null @@ -1,834 +0,0 @@ -from contextlib import contextmanager -from mock import PropertyMock, call -import re - -from django.core.exceptions import ValidationError -from django.test import TestCase -from django_mock_queries.mocks import mocked_relations - -from constants import datatypes -from metadata.models import CompoundDatatype, CompoundDatatypeMember, Datatype -from method.models import Method -from pipeline.models import Pipeline, PipelineFamily, PipelineStep,\ - PipelineStepInputCable, PipelineOutputCable, PipelineCable -from transformation.models import TransformationInput, XputStructure,\ - TransformationOutput, Transformation - - -@mocked_relations(Pipeline, - PipelineStep, - Method, - Transformation, - CompoundDatatype, - Datatype, - PipelineCable, - PipelineStepInputCable, - PipelineOutputCable, - XputStructure) -class PipelineMockTests(TestCase): - """Tests for basic Pipeline functionality.""" - def test_pipeline_no_inputs_no_steps(self): - """A Pipeline with no inputs and no steps is clean but not complete.""" - p = Pipeline(family=PipelineFamily()) - - p.clean() - - self.assertRaisesRegexp( - ValidationError, - re.escape("Pipeline {} has no steps".format(p)), - p.complete_clean - ) - - """Tests for basic Pipeline functionality.""" - def test_pipeline_one_valid_input_no_steps(self): - """A Pipeline with one valid input, but no steps, is clean but not complete.""" - p = Pipeline(family=PipelineFamily()) - self.add_inputs(p, TransformationInput(dataset_idx=1)) - - p.clean() - - self.assertRaisesRegexp( - ValidationError, - re.escape("Pipeline {} has no steps".format(p)), - p.complete_clean - ) - - def test_pipeline_one_invalid_input_clean(self): - """A Pipeline with one input not numbered "1" is not clean.""" - p = Pipeline(family=PipelineFamily()) - self.add_inputs(p, TransformationInput(dataset_idx=4)) - - error = "Inputs are not consecutively numbered starting from 1" - self.assertRaisesRegexp(ValidationError, error, p.clean) - self.assertRaisesRegexp(ValidationError, error, p.complete_clean) - - def test_pipeline_many_valid_inputs_clean(self): - """A Pipeline with multiple, properly indexed inputs is clean.""" - p = Pipeline(family=PipelineFamily()) - self.add_inputs(p, - TransformationInput(dataset_idx=2), - TransformationInput(dataset_idx=1), - TransformationInput(dataset_idx=3)) - - p.clean() - - def test_pipeline_many_invalid_inputs_clean(self): - """A Pipeline with multiple, badly indexed inputs is not clean.""" - p = Pipeline(family=PipelineFamily()) - self.add_inputs(p, - TransformationInput(dataset_idx=2), - TransformationInput(dataset_idx=3), - TransformationInput(dataset_idx=4)) - - self.assertRaisesRegexp( - ValidationError, - "Inputs are not consecutively numbered starting from 1", - p.clean) - - def test_pipeline_one_valid_step_clean(self): - """A Pipeline with one validly indexed step and input is clean. - - The PipelineStep and Pipeline are not complete unless there is a - cable in place. - """ - with self.create_valid_pipeline() as p: - step1 = p.steps.all()[0] - step1.cables_in.clear() - step1.inputs[0].dataset_name = "input" - - error = 'Input "input" to transformation at step 1 is not cabled' - step1.clean() - self.assertRaisesRegexp(ValidationError, error, step1.complete_clean) - p.clean() - self.assertRaisesRegexp(ValidationError, error, p.complete_clean) - - def test_pipeline_one_bad_step_clean(self): - """Test step index check, one badly-indexed step case.""" - with self.create_valid_pipeline() as p: - p.steps.all()[0].step_num = 10 - - self.assertRaisesRegexp( - ValidationError, - "Steps are not consecutively numbered starting from 1", - p.clean) - - def test_pipeline_many_valid_steps_clean(self): - """Test step index check, well-indexed multi-step case.""" - p = Pipeline(family=PipelineFamily()) - self.add_inputs(p, - TransformationInput(dataset_idx=1)) - m = Method() - self.add_inputs(m, - TransformationInput(dataset_idx=1)) - p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=2)) - p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=1)) - p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=3)) - - p.clean() - - def test_pipeline_many_invalid_steps_clean(self): - """Test step index check, badly-indexed multi-step case.""" - p = Pipeline(family=PipelineFamily()) - self.add_inputs(p, - TransformationInput(dataset_idx=1)) - m = Method() - self.add_inputs(m, - TransformationInput(dataset_idx=1)) - p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=1)) - p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=4)) - p.steps.add(PipelineStep(pipeline=p, transformation=m, step_num=5)) - - self.assertRaisesRegexp( - ValidationError, - "Steps are not consecutively numbered starting from 1", - p.clean) - - def test_pipeline_one_step_valid_cabling_clean(self): - """Test good step cabling, one-step pipeline.""" - with self.create_valid_pipeline() as p: - step1 = p.steps.all()[0] - cable = step1.cables_in.all()[0] - outcable = p.outcables.all()[0] - - cable.clean() - step1.clean() - step1.complete_clean() - outcable.clean() - p.clean() - p.complete_clean() - - def test_pipeline_oneStep_invalid_cabling_invalid_pipeline_input_clean(self): - """Bad cabling: step looks for input that does not belong to the pipeline.""" - with self.create_valid_pipeline() as p: - step1 = p.steps.all()[0] - cable = step1.cables_in.all()[0] - - unrelated_input = self.create_input(datatypes.STR_PK, dataset_idx=3) - cable.source = unrelated_input - - self.assertRaisesRegexp(ValidationError, - 'Pipeline does not have input ".*"', - cable.clean) - # The following are just the same as the above, propagated upwards through clean()s. - self.assertRaisesRegexp(ValidationError, - 'Pipeline does not have input ".*"', - step1.clean) - self.assertRaisesRegexp(ValidationError, - 'Pipeline does not have input ".*"', - step1.complete_clean) - self.assertRaisesRegexp(ValidationError, - 'Pipeline does not have input ".*"', - p.clean) - - def test_pipeline_oneStep_invalid_cabling_incorrect_cdt_clean(self): - """Bad cabling: input is of wrong CompoundDatatype.""" - p = Pipeline(family=PipelineFamily()) - self.add_inputs(p, self.create_input(datatypes.INT_PK, dataset_idx=1)) - m = Method() - self.add_inputs(m, self.create_input(datatypes.STR_PK, dataset_idx=1)) - - step1 = PipelineStep(pipeline=p, transformation=m, step_num=1) - p.steps.add(step1) - - cable = PipelineStepInputCable(pipelinestep=step1, - source_step=0, - source=p.inputs.all()[0], - dest=m.inputs.all()[0]) - cable.pipelinestepinputcable = cable - step1.cables_in.add(cable) - - cable.clean() - self.assertRaisesRegexp( - ValidationError, - 'Custom wiring required for cable "{}"'.format(cable), - cable.clean_and_completely_wired) - - def test_pipeline_oneStep_cabling_minrow_constraint_may_be_breached_clean(self): - """ Unverifiable cabling - - Step requests input with possibly too few rows (input min_row - unspecified). - """ - with self.create_valid_pipeline() as p: - step1 = p.steps.all()[0] - cable = step1.cables_in.all()[0] - method_input = step1.transformation.inputs[0] - method_input.structure.min_row = 10 - method_input.dataset_name = "input" - - # It's possible this step may have too few rows - self.assertRaisesRegexp( - ValidationError, - "Data fed to input \"input\" of step 1 may have too few rows", - cable.clean) - # This is just to check that the above propagated up. - self.assertRaisesRegexp( - ValidationError, - "Data fed to input \"input\" of step 1 may have too few rows", - p.clean) - - def test_pipeline_oneStep_cabling_minrow_constraints_may_breach_each_other_clean(self): - """ Unverifiable cabling - - Step requests input with possibly too few rows (input min_row specified). - """ - with self.create_valid_pipeline() as p: - step1 = p.steps.all()[0] - cable = step1.cables_in.all()[0] - p.inputs[0].structure.min_row = 5 - method_input = step1.transformation.inputs[0] - method_input.structure.min_row = 10 - method_input.dataset_name = "input" - - self.assertRaisesRegexp( - ValidationError, - "Data fed to input \"input\" of step 1 may have too few rows", - cable.clean) - self.assertRaisesRegexp( - ValidationError, - "Data fed to input \"input\" of step 1 may have too few rows", - p.clean) - - def test_pipeline_oneStep_cabling_maxRow_constraints_may_be_breached_clean(self): - """ Unverifiable cabling - - Step requests input with possibly too many rows (input max_row - unspecified) - """ - with self.create_valid_pipeline() as p: - step1 = p.steps.all()[0] - cable = step1.cables_in.all()[0] - method_input = step1.transformation.inputs[0] - method_input.structure.max_row = 10 - method_input.dataset_name = "input" - - # The pipeline input is unrestricted, but step 1 has max_row = 10 - self.assertRaisesRegexp( - ValidationError, - "Data fed to input \"input\" of step 1 may have too many rows", - cable.clean) - # Check propagation of error. - self.assertRaisesRegexp( - ValidationError, - "Data fed to input \"input\" of step 1 may have too many rows", - p.clean) - - def test_pipeline_oneStep_cabling_maxRow_constraints_may_breach_each_other_clean(self): - """ Unverifiable cabling - - Step requests input with possibly too many rows (max_row set for - pipeline input). - """ - with self.create_valid_pipeline() as p: - step1 = p.steps.all()[0] - cable = step1.cables_in.all()[0] - p.inputs[0].structure.max_row = 20 - method_input = step1.transformation.inputs[0] - method_input.structure.max_row = 10 - method_input.dataset_name = "input" - - # The pipeline max_row is not good enough to guarantee correctness - self.assertRaisesRegexp( - ValidationError, - "Data fed to input \"input\" of step 1 may have too many rows", - cable.clean) - self.assertRaisesRegexp( - ValidationError, - "Data fed to input \"input\" of step 1 may have too many rows", - p.clean) - - def test_pipeline_oneStep_outcable_references_nonexistent_step_clean(self): - """ Bad output cabling, request from nonexistent step. """ - with self.create_valid_pipeline() as p: - outcable = p.outcables[0] - outcable.source_step = 5 - - self.assertRaisesRegexp( - ValidationError, - "Output requested from a non-existent step", - outcable.clean) - # Check propagation of error. - self.assertRaisesRegexp( - ValidationError, - "Output requested from a non-existent step", - p.clean) - - def test_pipeline_oneStep_outcable_references_invalid_output_clean(self): - """Bad output cabling, request output not belonging to requested step""" - with self.create_valid_pipeline() as p: - unrelated_output = self.create_output(datatypes.STR_PK, dataset_idx=3) - m2 = Method() - m2.method = m2 - unrelated_output.transformation = m2 - outcable = p.outcables[0] - outcable.source = unrelated_output - - self.assertRaisesRegexp( - ValidationError, - 'Transformation at step 1 does not produce output ".*"', - outcable.clean) - self.assertRaisesRegexp( - ValidationError, - 'Transformation at step 1 does not produce output ".*"', - p.clean) - - def test_pipeline_oneStep_outcable_references_deleted_output_clean(self): - """Output cabling, one-step pipeline: request deleted step output (OK)""" - with self.create_valid_pipeline() as p: - step1 = p.steps.all()[0] - step1.outputs_to_delete.add(step1.outputs[0]) - outcable = p.outcables[0] - - outcable.clean() - p.clean() - - def test_pipeline_oneStep_bad_pipeline_output_indexing_clean(self): - """Bad output cabling, one-step pipeline: output not indexed 1""" - with self.create_valid_pipeline() as p: - outcable = p.outcables[0] - # Outcable references a valid step and output, but is itself badly indexed - outcable.output_idx = 9 - - outcable.clean() - self.assertRaisesRegexp( - ValidationError, - "Outputs are not consecutively numbered starting from 1", - p.clean) - - def test_pipeline_manySteps_valid_internal_cabling_clean(self): - """Test good step cabling, chained-step pipeline.""" - with self.create_valid_pipeline() as p: - step1 = p.steps.all()[0] - cable1 = step1.cables_in.all()[0] - - step2 = self.add_step(p) - cable2 = step2.cables_in.first() - - cable1.clean() - cable2.clean() - step1.clean() - step1.complete_clean() - step2.clean() - step2.complete_clean() - p.clean() - - def test_pipeline_manySteps_cabling_references_invalid_output_clean(self): - """Bad cabling: later step requests invalid input from previous.""" - with self.create_valid_pipeline() as p: - step2 = self.add_step(p) - cable = step2.cables_in.all()[0] - - unrelated_input = self.create_input(datatypes.STR_PK, dataset_idx=3) - cable.source = unrelated_input - - self.assertRaisesRegexp( - ValidationError, - 'Transformation at step 1 does not produce output ".*"', - cable.clean) - - # Check propagation of error. - self.assertRaisesRegexp( - ValidationError, - 'Transformation at step 1 does not produce output ".*"', - step2.clean) - self.assertRaisesRegexp( - ValidationError, - 'Transformation at step 1 does not produce output ".*"', - p.clean) - - def test_pipeline_manySteps_cabling_references_deleted_input_clean(self): - """Cabling: later step requests input deleted by producing step (OK).""" - with self.create_valid_pipeline() as p: - step1 = p.steps.all()[0] - step1.outputs_to_delete.add(step1.outputs[0]) - - step2 = self.add_step(p) - cable2 = step2.cables_in.all()[0] - - cable2.clean() - step2.clean() - p.clean() - - def test_pipeline_manySteps_cabling_references_incorrect_cdt_clean(self): - """Bad cabling: later step requests input of wrong CompoundDatatype.""" - with self.create_valid_pipeline() as p: - step2 = self.add_step(p) - cable2 = step2.cables_in.all()[0] - input_column = cable2.source.get_cdt().members.all()[0] - input_column.datatype.id = datatypes.INT_PK # should be STR_PK - - cable2.clean() - error_msg = 'Custom wiring required for cable "{}"'.format(str(cable2)) - self.assertRaisesRegexp(ValidationError, - error_msg, - cable2.clean_and_completely_wired) - self.assertRaisesRegexp(ValidationError, - error_msg, - step2.clean) - self.assertRaisesRegexp(ValidationError, - error_msg, - p.clean) - - def test_pipeline_manySteps_minRow_constraint_may_be_breached_clean(self): - """ Unverifiable cabling - - Later step requests input with possibly too few rows (min_row unset for - providing step). - """ - with self.create_valid_pipeline() as p: - step2 = self.add_step(p) - cable = step2.cables_in.all()[0] - method_input = step2.transformation.inputs[0] - method_input.structure.min_row = 10 - method_input.dataset_name = "input" - - self.assertRaisesRegexp( - ValidationError, - "Data fed to input \"input\" of step 2 may have too few rows", - cable.clean) - self.assertRaisesRegexp( - ValidationError, - "Data fed to input \"input\" of step 2 may have too few rows", - p.clean) - - def test_pipeline_manySteps_minrow_constraints_may_breach_each_other_clean(self): - """ Bad cabling: later step requests input with possibly too few rows. - - (providing step min_row is set) - """ - with self.create_valid_pipeline() as p: - step1 = p.steps.all()[0] - step2 = self.add_step(p) - cable = step2.cables_in.all()[0] - prev_output = step1.transformation.outputs[0] - prev_output.structure.min_row = 5 - method_input = step2.transformation.inputs[0] - method_input.structure.min_row = 10 - method_input.dataset_name = "input" - - self.assertRaisesRegexp( - ValidationError, - "Data fed to input \"input\" of step 2 may have too few rows", - cable.clean) - self.assertRaisesRegexp( - ValidationError, - "Data fed to input \"input\" of step 2 may have too few rows", - p.clean) - - def test_pipeline_manySteps_maxRow_constraint_may_be_breached_clean(self): - """ Bad cabling: later step requests input with possibly too many rows. - - (max_row unset for providing step) - """ - with self.create_valid_pipeline() as p: - step2 = self.add_step(p) - cable = step2.cables_in.all()[0] - method_input = step2.transformation.inputs[0] - method_input.structure.max_row = 100 - method_input.dataset_name = "input" - - self.assertRaisesRegexp( - ValidationError, - "Data fed to input \"input\" of step 2 may have too many rows", - cable.clean) - self.assertRaisesRegexp( - ValidationError, - "Data fed to input \"input\" of step 2 may have too many rows", - p.clean) - - def test_pipeline_manySteps_cabling_maxRow_constraints_may_breach_each_other_clean(self): - """ Bad cabling: later step requests input with possibly too many rows. - - (max_row for providing step is set) - """ - with self.create_valid_pipeline() as p: - step1 = p.steps.all()[0] - step2 = self.add_step(p) - cable = step2.cables_in.all()[0] - prev_output = step1.transformation.outputs[0] - prev_output.structure.max_row = 100 - method_input = step2.transformation.inputs[0] - method_input.structure.max_row = 50 - method_input.dataset_name = "input" - self.assertRaisesRegexp( - ValidationError, - "Data fed to input \"input\" of step 2 may have too many rows", - cable.clean) - self.assertRaisesRegexp( - ValidationError, - "Data fed to input \"input\" of step 2 may have too many rows", - p.clean) - - def test_pipeline_manySteps_valid_outcable_clean(self): - """Good output cabling, chained-step pipeline.""" - with self.create_valid_pipeline() as p: - self.add_step(p) - - outcable1, outcable2 = p.outcables.all() - - outcable1.clean() - outcable2.clean() - p.clean() - - def test_pipeline_manySteps_outcable_references_nonexistent_step_clean(self): - """Bad output cabling, chained-step pipeline: request from nonexistent step""" - with self.create_valid_pipeline() as p: - self.add_step(p) - - outcable1, outcable2 = p.outcables.all() - outcable1.source_step = 5 - - self.assertRaisesRegexp( - ValidationError, - "Output requested from a non-existent step", - outcable1.clean) - outcable2.clean() - self.assertRaisesRegexp( - ValidationError, - "Output requested from a non-existent step", - p.clean) - - def test_pipeline_manySteps_outcable_references_invalid_output_clean(self): - """Bad output cabling, chained-step pipeline: request output not belonging to requested step""" - with self.create_valid_pipeline() as p: - self.add_step(p) - - outcable1, outcable2 = p.outcables.all() - unrelated_output = self.create_output(datatypes.STR_PK, dataset_idx=3) - m3 = Method() - m3.method = m3 - unrelated_output.transformation = m3 - outcable2.source = unrelated_output - - self.assertEquals(outcable1.clean(), None) - self.assertRaisesRegexp( - ValidationError, - 'Transformation at step 2 does not produce output ".*"', - outcable2.clean) - self.assertRaisesRegexp( - ValidationError, - 'Transformation at step 2 does not produce output ".*"', - p.clean) - - def test_pipeline_manySteps_outcable_references_deleted_output_clean(self): - """Output cabling, chained-step pipeline: request deleted step output (OK)""" - with self.create_valid_pipeline() as p: - step2 = self.add_step(p) - step2.outputs_to_delete.add(step2.outputs[0]) - - outcable1, outcable2 = p.outcables.all() - - outcable1.clean() - outcable2.clean() - p.clean() - - def test_pipeline_manySteps_outcable_references_invalid_output_index_clean(self): - """Bad output cabling, chain-step pipeline: outputs not consecutively numbered starting from 1""" - with self.create_valid_pipeline() as p: - self.add_step(p) - - outcable1, outcable2 = p.outcables.all() - outcable2.output_idx = 5 - - outcable1.clean() - outcable2.clean() - self.assertRaisesRegexp( - ValidationError, - "Outputs are not consecutively numbered starting from 1", - p.clean) - - def test_pipeline_with_1_step_and_2_inputs_both_cabled_good(self): - """ Pipeline with 1 step with 2 inputs / 1 output - - Both inputs are cabled (good) - """ - with self.create_valid_pipeline() as p: - step1 = p.steps.all()[0] - m = step1.transformation - cable1 = step1.cables_in.all()[0] - source = self.create_input(datatypes.STR_PK, dataset_idx=2) - self.add_inputs(p, source) - dest = self.create_input(datatypes.STR_PK, dataset_idx=2) - self.add_inputs(m, dest) - cable2 = PipelineStepInputCable(pipelinestep=step1, - source_step=0, - source=source, - dest=dest) - cable2.pipelinestepinputcable = cable2 - step1.cables_in.add(cable2) - - cable1.clean() - cable2.clean() - step1.clean() - step1.complete_clean() - p.clean() - - def test_pipeline_with_1_step_and_2_inputs_cabled_more_than_once_bad(self): - """ Pipeline with 1 step with 2 inputs / 1 output - - input 2 is cabled twice (bad) - """ - with self.create_valid_pipeline() as p: - step1 = p.steps.all()[0] - m = step1.transformation - cable1 = step1.cables_in.all()[0] - source = self.create_input(datatypes.STR_PK, dataset_idx=2) - self.add_inputs(p, source) - dest = self.create_input(datatypes.STR_PK, - dataset_idx=2, - dataset_name="r") - self.add_inputs(m, dest) - cable2 = PipelineStepInputCable(pipelinestep=step1, - source_step=0, - source=source, - dest=dest) - cable2.pipelinestepinputcable = cable2 - step1.cables_in.add(cable2) - cable3 = PipelineStepInputCable(pipelinestep=step1, - source_step=0, - source=source, - dest=dest) - cable3.pipelinestepinputcable = cable3 - step1.cables_in.add(cable3) - - cable1.clean() - cable2.clean() - cable3.clean() - - self.assertRaisesRegexp( - ValidationError, - "Input \"r\" to transformation at step 1 is cabled more than once", - step1.clean) - self.assertRaisesRegexp( - ValidationError, - "Input \"r\" to transformation at step 1 is cabled more than once", - step1.complete_clean) - self.assertRaisesRegexp( - ValidationError, - "Input \"r\" to transformation at step 1 is cabled more than once", - p.clean) - - def test_pipeline_with_1_step_and_2_inputs_but_only_first_input_is_cabled_in_step_1_bad(self): - """ Pipeline with 1 step with 2 inputs / 1 output - - Only the first input is cabled (bad) - """ - with self.create_valid_pipeline() as p: - step1 = p.steps.all()[0] - m = step1.transformation - source = self.create_input(datatypes.STR_PK, dataset_idx=2) - self.add_inputs(p, source) - dest = self.create_input(datatypes.STR_PK, - dataset_idx=2, - dataset_name="r") - self.add_inputs(m, dest) - - # Step is clean (cables are OK) but not complete (inputs not quenched). - step1.clean() - self.assertRaisesRegexp( - ValidationError, - "Input \"r\" to transformation at step 1 is not cabled", - step1.complete_clean) - - def test_create_outputs(self): - """ - Create outputs from output cablings; also change the output cablings - and recreate the outputs to see if they're correct. - """ - with self.create_valid_pipeline() as p: - p.outcables.all()[0].output_name = 'step1_out' - Transformation.outputs = PropertyMock('Transformation.outputs') - Transformation.outputs.create.return_value = TransformationOutput() - - p.create_outputs() - - self.assertEqual( - [call(y=0, x=0, dataset_idx=1, dataset_name='step1_out')], - p.outputs.create.call_args_list) - # noinspection PyUnresolvedReferences - self.assertEqual(1, XputStructure.save.call_count) - - def test_create_outputs_multi_step(self): - """Testing create_outputs with a multi-step pipeline.""" - with self.create_valid_pipeline() as p: - self.add_step(p) - p.outcables.all()[0].output_name = 'step1_out' - p.outcables.all()[1].output_name = 'step2_out' - Transformation.outputs = PropertyMock('Transformation.outputs') - Transformation.outputs.create.return_value = TransformationOutput() - - p.create_outputs() - - self.assertEqual( - [call(y=0, x=0, dataset_idx=1, dataset_name='step1_out'), - call(y=0, x=0, dataset_idx=2, dataset_name='step2_out')], - p.outputs.create.call_args_list) - # noinspection PyUnresolvedReferences - self.assertEqual(2, XputStructure.save.call_count) - - @contextmanager - def create_valid_pipeline(self): - p = Pipeline(family=PipelineFamily()) - self.add_inputs(p, self.create_input(datatypes.STR_PK, dataset_idx=1)) - m = Method() - m.method = m - self.add_inputs(m, self.create_input(datatypes.STR_PK, dataset_idx=1)) - self.add_outputs(m, self.create_output(datatypes.STR_PK, dataset_idx=1)) - - step1 = PipelineStep(pipeline=p, transformation=m, step_num=1) - p.steps.add(step1) - - cable = PipelineStepInputCable(pipelinestep=step1, - source_step=0, - source=p.inputs.all()[0], - dest=m.inputs.all()[0]) - cable.pipelinestepinputcable = cable - step1.cables_in.add(cable) - - outcable = PipelineOutputCable( - pipeline=p, - output_idx=1, - source_step=1, - source=m.outputs.all()[0], - output_cdt=m.outputs.all()[0].get_cdt()) - p.outcables.add(outcable) - - yield p - - def add_step(self, pipeline): - prev_step = pipeline.steps[-1] - m = Method() - m.method = m - self.add_inputs(m, self.create_input(datatypes.STR_PK, dataset_idx=1)) - self.add_outputs(m, self.create_output(datatypes.STR_PK, dataset_idx=1)) - step = PipelineStep(pipeline=pipeline, - transformation=m, - step_num=prev_step.step_num + 1) - pipeline.steps.add(step) - - cable = PipelineStepInputCable( - pipelinestep=step, - source_step=prev_step.step_num, - source=prev_step.transformation.outputs[0], - dest=m.inputs[0]) - cable.pipelinestepinputcable = cable - step.cables_in.add(cable) - outcable = PipelineOutputCable( - pipeline=pipeline, - output_idx=step.step_num, - source_step=step.step_num, - source=m.outputs[0], - output_cdt=m.outputs[0].get_cdt()) - pipeline.outcables.add(outcable) - - return step - - def create_input(self, *column_datatype_ids, **kwargs): - new_input = TransformationInput(**kwargs) - new_input.transformationinput = new_input - self.set_structure(new_input, column_datatype_ids) - return new_input - - def create_output(self, *column_datatype_ids, **kwargs): - new_output = TransformationOutput(**kwargs) - new_output.transformationoutput = new_output - self.set_structure(new_output, column_datatype_ids) - return new_output - - def set_structure(self, xput, column_datatype_ids): - if column_datatype_ids: - cdt = CompoundDatatype() - for datatype_id in column_datatype_ids: - cdt.members.add(CompoundDatatypeMember( - datatype=Datatype(id=datatype_id))) - xput.structure = XputStructure(compounddatatype=cdt) - - def add_inputs(self, transformation, *inputs): - """ Wire up the inputs to a mocked transformation. - - :param Transformation transformation: inputs will be added to this - """ - for t_input in inputs: - t_input.transformationinput = t_input - t_input.transformation = transformation - transformation.inputs.add(t_input) - - def add_outputs(self, transformation, *outputs): - """ Wire up the outputs to a mocked transformation. - - :param Transformation transformation: outputs will be added to this - """ - for t_output in outputs: - t_output.transformationoutput = t_output - t_output.transformation = transformation - transformation.outputs.add(t_output) - - -class PipelineUpdateMockTests(TestCase): - @mocked_relations(Pipeline) - def test_no_steps(self): - pipeline = Pipeline() - - updates = pipeline.find_step_updates() - - self.assertEqual([], updates) diff --git a/kive/portal/fixtures/archive_no_runs_test_environment.json b/kive/portal/fixtures/archive_no_runs_test_environment.json deleted file mode 100644 index d5bcb3c60..000000000 --- a/kive/portal/fixtures/archive_no_runs_test_environment.json +++ /dev/null @@ -1,2558 +0,0 @@ -[ - { - "fields": { - "datatype": 8, - "rule": "^[ACGTacgt]*$", - "ruletype": "regexp" - }, - "model": "metadata.basicconstraint", - "pk": 6 - }, - { - "fields": { - "datatype": 9, - "rule": "^[ACGUacgu]*$", - "ruletype": "regexp" - }, - "model": "metadata.basicconstraint", - "pk": 7 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "label", - "compounddatatype": 5, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 7 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "integer", - "compounddatatype": 5, - "datatype": 4 - }, - "model": "metadata.compounddatatypemember", - "pk": 8 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "float", - "compounddatatype": 5, - "datatype": 3 - }, - "model": "metadata.compounddatatypemember", - "pk": 9 - }, - { - "fields": { - "blankable": false, - "column_idx": 4, - "column_name": "bool", - "compounddatatype": 5, - "datatype": 2 - }, - "model": "metadata.compounddatatypemember", - "pk": 10 - }, - { - "fields": { - "blankable": false, - "column_idx": 5, - "column_name": "rna", - "compounddatatype": 5, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 11 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "label", - "compounddatatype": 6, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 12 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "label", - "compounddatatype": 7, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 13 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "PBMCseq", - "compounddatatype": 7, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 14 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "PLAseq", - "compounddatatype": 7, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 15 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "SeqToComplement", - "compounddatatype": 8, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 16 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "ComplementedSeq", - "compounddatatype": 9, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 17 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "SeqToComplement", - "compounddatatype": 10, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 18 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "ComplementedSeq", - "compounddatatype": 11, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 19 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "x", - "compounddatatype": 12, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 20 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "y", - "compounddatatype": 12, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 21 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "k", - "compounddatatype": 13, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 22 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "a", - "compounddatatype": 14, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 23 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "b", - "compounddatatype": 14, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 24 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "c", - "compounddatatype": 14, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 25 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "a^2", - "compounddatatype": 15, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 26 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "b^2", - "compounddatatype": 15, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 27 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "c^2", - "compounddatatype": 15, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 28 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "StrCol1", - "compounddatatype": 16, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 29 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "DNACol2", - "compounddatatype": 16, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 30 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "StrCol3", - "compounddatatype": 16, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 31 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "a", - "compounddatatype": 17, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 32 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "b", - "compounddatatype": 17, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 33 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "c", - "compounddatatype": 17, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 34 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "x", - "compounddatatype": 18, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 35 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "y", - "compounddatatype": 18, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 36 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "word", - "compounddatatype": 19, - "datatype": 10 - }, - "model": "metadata.compounddatatypemember", - "pk": 37 - }, - { - "fields": { - "compounddatatype": 14, - "dataset": 1, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 2, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 13, - "dataset": 3, - "num_rows": 13 - }, - "model": "librarian.datasetstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 13, - "dataset": 4, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 6, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 5 - }, - { - "fields": { - "compounddatatype": 14, - "dataset": 7, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 6 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 8, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 7 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 9, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 8 - }, - { - "fields": { - "compounddatatype": 13, - "dataset": 10, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 9 - }, - { - "fields": { - "compounddatatype": 14, - "dataset": 13, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 10 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 14, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 11 - }, - { - "fields": { - "compounddatatype": 17, - "dataset": 15, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 12 - }, - { - "fields": { - "compounddatatype": 18, - "dataset": 16, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 13 - }, - { - "fields": { - "compounddatatype": 18, - "dataset": 17, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 14 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 4, - "step_num": 1, - "transformation": 2, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 1 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 1, - "transformation": 1, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 2 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 2, - "transformation": 4, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 3 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 3, - "transformation": 3, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 4 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 1 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 2 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 3 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 4 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 5 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 6 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 7 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 8 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 9 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 10 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 11 - }, - { - "fields": { - "dest": 3, - "keep_output": false, - "pipelinestep": 1, - "source": 11, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 1 - }, - { - "fields": { - "dest": 4, - "keep_output": false, - "pipelinestep": 1, - "source": 12, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 2 - }, - { - "fields": { - "dest": 1, - "keep_output": false, - "pipelinestep": 2, - "source": 15, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 4 - }, - { - "fields": { - "dest": 11, - "keep_output": false, - "pipelinestep": 3, - "source": 13, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 5 - }, - { - "fields": { - "dest": 12, - "keep_output": false, - "pipelinestep": 3, - "source": 14, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 6 - }, - { - "fields": { - "dest": 7, - "keep_output": false, - "pipelinestep": 4, - "source": 2, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 7 - }, - { - "fields": { - "dest": 6, - "keep_output": false, - "pipelinestep": 4, - "source": 16, - "source_step": 2 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 8 - }, - { - "fields": { - "cable": 5, - "dest_pin": 21, - "source_pin": 23 - }, - "model": "pipeline.customcablewire", - "pk": 1 - }, - { - "fields": { - "cable": 5, - "dest_pin": 20, - "source_pin": 25 - }, - "model": "pipeline.customcablewire", - "pk": 2 - }, - { - "fields": { - "cable": 7, - "dest_pin": 21, - "source_pin": 20 - }, - "model": "pipeline.customcablewire", - "pk": 3 - }, - { - "fields": { - "cable": 7, - "dest_pin": 20, - "source_pin": 21 - }, - "model": "pipeline.customcablewire", - "pk": 4 - }, - { - "fields": { - "cable": 9, - "dest_pin": 21, - "source_pin": 24 - }, - "model": "pipeline.customcablewire", - "pk": 5 - }, - { - "fields": { - "cable": 9, - "dest_pin": 20, - "source_pin": 25 - }, - "model": "pipeline.customcablewire", - "pk": 6 - }, - { - "fields": { - "output_cdt": 14, - "output_idx": 1, - "output_name": "D1_out", - "pipeline": 4, - "source": 5, - "source_step": 1 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 3 - }, - { - "fields": { - "output_cdt": 12, - "output_idx": 1, - "output_name": "E1_out", - "pipeline": 5, - "source": 16, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 9 - }, - { - "fields": { - "output_cdt": 13, - "output_idx": 2, - "output_name": "E2_out", - "pipeline": 5, - "source": 8, - "source_step": 3 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 10 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 3, - "output_name": "E3_rawout", - "pipeline": 5, - "source": 10, - "source_step": 3 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 11 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 1 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 2 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 3 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 4 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 5 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 6 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 7 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 8 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 9 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 10 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 11 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 12 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 13 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 14 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 15 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 16 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 17 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 18 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 19 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 20 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 21 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 22 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 23 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 24 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 25 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 2 - }, - "model": "transformation.xputstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 3 - }, - "model": "transformation.xputstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 4 - }, - "model": "transformation.xputstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": 5, - "min_row": null, - "transf_xput": 5 - }, - "model": "transformation.xputstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": null, - "min_row": null, - "transf_xput": 6 - }, - "model": "transformation.xputstructure", - "pk": 5 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 7 - }, - "model": "transformation.xputstructure", - "pk": 6 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 8 - }, - "model": "transformation.xputstructure", - "pk": 7 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 11 - }, - "model": "transformation.xputstructure", - "pk": 8 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 12 - }, - "model": "transformation.xputstructure", - "pk": 9 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": null, - "min_row": null, - "transf_xput": 13 - }, - "model": "transformation.xputstructure", - "pk": 10 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": 10, - "transf_xput": 14 - }, - "model": "transformation.xputstructure", - "pk": 11 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": 5, - "min_row": null, - "transf_xput": 16 - }, - "model": "transformation.xputstructure", - "pk": 12 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": 5, - "min_row": null, - "transf_xput": 17 - }, - "model": "transformation.xputstructure", - "pk": 13 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 18 - }, - "model": "transformation.xputstructure", - "pk": 14 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 20 - }, - "model": "transformation.xputstructure", - "pk": 15 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 21 - }, - "model": "transformation.xputstructure", - "pk": 16 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 22 - }, - "model": "transformation.xputstructure", - "pk": 17 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 23 - }, - "model": "transformation.xputstructure", - "pk": 18 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "A1_rawin", - "transformation": 1 - }, - "model": "transformation.transformationinput", - "pk": 1 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "B1_in", - "transformation": 2 - }, - "model": "transformation.transformationinput", - "pk": 3 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "B2_in", - "transformation": 2 - }, - "model": "transformation.transformationinput", - "pk": 4 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "C1_in", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 6 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "C2_in", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 7 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "D1_in", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 11 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "D2_in", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 12 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "E1_in", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 13 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "E2_in", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 14 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "E3_rawin", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 15 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "strings", - "transformation": 6 - }, - "model": "transformation.transformationinput", - "pk": 20 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "strings", - "transformation": 7 - }, - "model": "transformation.transformationinput", - "pk": 22 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "raw", - "transformation": 8 - }, - "model": "transformation.transformationinput", - "pk": 24 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "A1_out", - "transformation": 1 - }, - "model": "transformation.transformationoutput", - "pk": 2 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "B1_out", - "transformation": 2 - }, - "model": "transformation.transformationoutput", - "pk": 5 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "C1_out", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 8 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "C2_rawout", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 9 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "C3_rawout", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 10 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "D1_out", - "transformation": 4 - }, - "model": "transformation.transformationoutput", - "pk": 16 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "E1_out", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 17 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "E2_out", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 18 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "E3_rawout", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 19 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "same_strings", - "transformation": 6 - }, - "model": "transformation.transformationoutput", - "pk": 21 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "untouched_strings", - "transformation": 7 - }, - "model": "transformation.transformationoutput", - "pk": 23 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "same_raw", - "transformation": 8 - }, - "model": "transformation.transformationoutput", - "pk": 25 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:00Z", - "email": "lennon@thebeatles.com", - "first_name": "", - "groups": [ - 1 - ], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$xpQDP9BA2PZV$eJImlRNFGyJ9ufHpltoDS0mjkNBo4qJXDwt1J2+492g=", - "user_permissions": [], - "username": "john" - }, - "model": "auth.user", - "pk": 2 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:00Z", - "email": "starr@thebeatles.com", - "first_name": "", - "groups": [], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$wB30U4u4H3kj$+AMeA45K9O30ovTr9ohGA3EPxwBZg4selSYQdHLs0Yg=", - "user_permissions": [], - "username": "ringo" - }, - "model": "auth.user", - "pk": 3 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:02Z", - "email": "bob@talabs.com", - "first_name": "", - "groups": [ - 1 - ], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$abeTfFPdzsgV$8iBSeaFGnE1jl5oAlkDjnnNSQXCD2TsOhQoOtDFQAw4=", - "user_permissions": [], - "username": "bob" - }, - "model": "auth.user", - "pk": 4 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:00Z", - "description": "String consisting of ACGTacgt", - "groups_allowed": [ - 1 - ], - "name": "DNANucSeq", - "prototype": null, - "restricts": [ - 1 - ], - "user": 2, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 8 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:00Z", - "description": "String consisting of ACGUacgu", - "groups_allowed": [ - 1 - ], - "name": "RNANucSeq", - "prototype": null, - "restricts": [ - 1 - ], - "user": 2, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 9 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:02Z", - "description": "sequences of ASCII characters", - "groups_allowed": [ - 1 - ], - "name": "my_string", - "prototype": null, - "restricts": [ - 1 - ], - "user": 4, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 10 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [], - "name": "", - "user": 2, - "users_allowed": [ - 3 - ] - }, - "model": "metadata.compounddatatype", - "pk": 6 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 7 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 8 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 9 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 10 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 11 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 12 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 13 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 14 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 15 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 16 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 17 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 18 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 4, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 19 - }, - { - "fields": { - "MD5_checksum": "ab7afe2f453d27fd168641a1f9271f26", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_triplet.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:06.348Z", - "name": "triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "02d5b66858d003b81ae3dc712b74e85c", - "_redacted": false, - "dataset_file": "Datasets/2017_04/doublet_cdt.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:06.415Z", - "name": "doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "29feb269edcac613d6eea432c14640bd", - "_redacted": false, - "dataset_file": "Datasets/2017_04/singlet_cdt_large.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:06.482Z", - "name": "singlet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 3 - }, - { - "fields": { - "MD5_checksum": "00fd50bfe7ac63e2165a442f4101e05c", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_singlet.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:06.535Z", - "name": "singlet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 4 - }, - { - "fields": { - "MD5_checksum": "7dc85e11b5c02e434af5bd3b3da9938e", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_raw.fasta", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:06.577Z", - "name": "raw_DS", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 5 - }, - { - "fields": { - "MD5_checksum": "542676b23e121d16db8d41ccdae65fd1", - "_redacted": false, - "dataset_file": "", - "date_created": "2000-01-01T00:00:01Z", - "description": "", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:06.611Z", - "name": "D1_in_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 6 - }, - { - "fields": { - "MD5_checksum": "56f9e5585b9d699829e2fda931d944f8", - "_redacted": false, - "dataset_file": "Datasets/2017_04/C1_in_triplet.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "triplet 3 rows", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:06.662Z", - "name": "C1_in_triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 7 - }, - { - "fields": { - "MD5_checksum": "10a4bd15d1300fcaf5a3bdad54ef1d28", - "_redacted": false, - "dataset_file": "", - "date_created": "2000-01-01T00:00:01Z", - "description": "", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:06.729Z", - "name": "C2_in_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 8 - }, - { - "fields": { - "MD5_checksum": "10a4bd15d1300fcaf5a3bdad54ef1d28", - "_redacted": false, - "dataset_file": "Datasets/2017_04/E11_32_output.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "result of E11_32 fed by doublet_cdt.csv", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:06.779Z", - "name": "E11_32 output doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 9 - }, - { - "fields": { - "MD5_checksum": "00fd50bfe7ac63e2165a442f4101e05c", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_singlet_bbebLoZ.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:06.838Z", - "name": "raw", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 10 - }, - { - "fields": { - "MD5_checksum": "7dc85e11b5c02e434af5bd3b3da9938e", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_raw_tmCiC4V.fasta", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:06.878Z", - "name": "C2_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 11 - }, - { - "fields": { - "MD5_checksum": "7dc85e11b5c02e434af5bd3b3da9938e", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_raw_OmSCZlb.fasta", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:06.901Z", - "name": "C3_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 12 - }, - { - "fields": { - "MD5_checksum": "56f9e5585b9d699829e2fda931d944f8", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_triplet_3_rows.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:06.940Z", - "name": "triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 13 - }, - { - "fields": { - "MD5_checksum": "83957f8aea1c75da9f7fbf9bc90da979", - "_redacted": false, - "dataset_file": "Datasets/2017_04/doublet_remuxed_from_t3r.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "doublet remuxed from triplet", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:07.007Z", - "name": "E1_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 14 - }, - { - "fields": { - "MD5_checksum": "d83866e4c23cd9503aa2f9fc565502b5", - "_redacted": false, - "dataset_file": "Datasets/2017_04/DNA_triplet.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "DNA triplet data", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:07.075Z", - "name": "DNA_triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 15 - }, - { - "fields": { - "MD5_checksum": "9eab0dfed5fcc89bd581f61995816ef0", - "_redacted": false, - "dataset_file": "Datasets/2017_04/E01_21_DNA_doublet.csv", - "date_created": "2000-01-01T00:00:02Z", - "description": "DNA doublet data coming from DNA_triplet.csv but remultiplexed according to cable E01_21", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:07.204Z", - "name": "E01_21_DNA_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 16 - }, - { - "fields": { - "MD5_checksum": "9b55b9431da6dba74ea61edf8e2ac044", - "_redacted": false, - "dataset_file": "Datasets/2017_04/E21_41_DNA_doublet.csv", - "date_created": "2000-01-01T00:00:02Z", - "description": "DNA doublet data coming from DNA_triplet.csv but remultiplexed according to cable E21_41", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:07.320Z", - "name": "E21_41_DNA_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 17 - }, - { - "fields": { - "description": "Just a CR", - "filename": "generic_script.py", - "groups_allowed": [ - 1 - ], - "name": "genericCR", - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 1 - }, - { - "fields": { - "description": "a script to do nothing", - "filename": "noop.sh", - "groups_allowed": [ - 1 - ], - "name": "noop", - "user": 4, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "e353a62fe8cb6a56d31b8c65d70b217d", - "coderesource": 1, - "content_file": "CodeResources/generic_script.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "desc", - "revision_name": "v1", - "revision_number": 1, - "revision_parent": null, - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "d218c00725d473639f347d456de381d8", - "coderesource": 2, - "content_file": "CodeResources/fdopen", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "1", - "revision_number": 1, - "revision_parent": null, - "user": 4, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 2 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 1 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 2, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 2 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 3, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 3 - }, - { - "fields": { - "driver": 2, - "family": 2, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 6 - }, - { - "fields": { - "driver": 2, - "family": 3, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 7 - }, - { - "fields": { - "driver": 2, - "family": 4, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 8 - }, - { - "fields": { - "description": "Holds methods A/B/C", - "groups_allowed": [ - 1 - ], - "name": "method_family", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 1 - }, - { - "fields": { - "description": "a method to do nothing to strings", - "groups_allowed": [ - 1 - ], - "name": "string noop", - "user": 4, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 2 - }, - { - "fields": { - "description": "a TOTALLY DIFFERENT method that TOTALLY does SOMETHING to strings by leaving them alone", - "groups_allowed": [ - 1 - ], - "name": "string trivial", - "user": 4, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 3 - }, - { - "fields": { - "description": "do nothing to raw data", - "groups_allowed": [ - 1 - ], - "name": "raw noop", - "user": 4, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 4 - }, - { - "fields": { - "description": "PF desc", - "groups_allowed": [ - 1 - ], - "name": "Pipeline_family", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 1 - }, - { - "fields": { - "family": 1, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 4 - }, - { - "fields": { - "family": 1, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 2, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "A_desc", - "revision_name": "mA_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 1 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "B_desc", - "revision_name": "mB_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 2 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "C_desc", - "revision_name": "mC_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 3 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "D", - "revision_name": "pD_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 4 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "E", - "revision_name": "pE_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 4, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 6 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 4, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 7 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 4, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 8 - }, - { - "fields": { - "dataset": 1, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 3, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 3 - }, - { - "fields": { - "dataset": 4, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 4 - }, - { - "fields": { - "dataset": 6, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 5 - }, - { - "fields": { - "dataset": 7, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 6 - }, - { - "fields": { - "dataset": 8, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 7 - }, - { - "fields": { - "dataset": 9, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 8 - }, - { - "fields": { - "dataset": 10, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 9 - }, - { - "fields": { - "dataset": 13, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 10 - }, - { - "fields": { - "dataset": 14, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 11 - }, - { - "fields": { - "dataset": 15, - "end_time": "2000-01-01T00:00:02Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 12 - }, - { - "fields": { - "dataset": 16, - "end_time": "2000-01-01T00:00:02Z", - "execlog": null, - "start_time": "2000-01-01T00:00:02Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 13 - }, - { - "fields": { - "dataset": 17, - "end_time": "2000-01-01T00:00:02Z", - "execlog": null, - "start_time": "2000-01-01T00:00:02Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 14 - } -] \ No newline at end of file diff --git a/kive/portal/fixtures/archive_test_environment.json b/kive/portal/fixtures/archive_test_environment.json deleted file mode 100644 index e391965a2..000000000 --- a/kive/portal/fixtures/archive_test_environment.json +++ /dev/null @@ -1,2905 +0,0 @@ -[ - { - "fields": { - "datatype": 8, - "rule": "^[ACGTacgt]*$", - "ruletype": "regexp" - }, - "model": "metadata.basicconstraint", - "pk": 6 - }, - { - "fields": { - "datatype": 9, - "rule": "^[ACGUacgu]*$", - "ruletype": "regexp" - }, - "model": "metadata.basicconstraint", - "pk": 7 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "label", - "compounddatatype": 5, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 7 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "integer", - "compounddatatype": 5, - "datatype": 4 - }, - "model": "metadata.compounddatatypemember", - "pk": 8 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "float", - "compounddatatype": 5, - "datatype": 3 - }, - "model": "metadata.compounddatatypemember", - "pk": 9 - }, - { - "fields": { - "blankable": false, - "column_idx": 4, - "column_name": "bool", - "compounddatatype": 5, - "datatype": 2 - }, - "model": "metadata.compounddatatypemember", - "pk": 10 - }, - { - "fields": { - "blankable": false, - "column_idx": 5, - "column_name": "rna", - "compounddatatype": 5, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 11 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "label", - "compounddatatype": 6, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 12 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "label", - "compounddatatype": 7, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 13 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "PBMCseq", - "compounddatatype": 7, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 14 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "PLAseq", - "compounddatatype": 7, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 15 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "SeqToComplement", - "compounddatatype": 8, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 16 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "ComplementedSeq", - "compounddatatype": 9, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 17 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "SeqToComplement", - "compounddatatype": 10, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 18 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "ComplementedSeq", - "compounddatatype": 11, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 19 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "x", - "compounddatatype": 12, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 20 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "y", - "compounddatatype": 12, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 21 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "k", - "compounddatatype": 13, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 22 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "a", - "compounddatatype": 14, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 23 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "b", - "compounddatatype": 14, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 24 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "c", - "compounddatatype": 14, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 25 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "a^2", - "compounddatatype": 15, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 26 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "b^2", - "compounddatatype": 15, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 27 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "c^2", - "compounddatatype": 15, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 28 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "StrCol1", - "compounddatatype": 16, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 29 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "DNACol2", - "compounddatatype": 16, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 30 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "StrCol3", - "compounddatatype": 16, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 31 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "a", - "compounddatatype": 17, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 32 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "b", - "compounddatatype": 17, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 33 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "c", - "compounddatatype": 17, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 34 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "x", - "compounddatatype": 18, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 35 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "y", - "compounddatatype": 18, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 36 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "word", - "compounddatatype": 19, - "datatype": 10 - }, - "model": "metadata.compounddatatypemember", - "pk": 37 - }, - { - "fields": { - "dataset": 1, - "index": 1, - "run": 2 - }, - "model": "archive.runinput", - "pk": 1 - }, - { - "fields": { - "dataset": 3, - "index": 2, - "run": 2 - }, - "model": "archive.runinput", - "pk": 2 - }, - { - "fields": { - "dataset": 5, - "index": 3, - "run": 2 - }, - "model": "archive.runinput", - "pk": 3 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 1, - "end_time": null, - "execrecord": 1, - "reused": false, - "start_time": null - }, - "model": "archive.runcomponent", - "pk": 1 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 1, - "end_time": null, - "execrecord": 2, - "reused": false, - "start_time": null - }, - "model": "archive.runcomponent", - "pk": 2 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 1, - "end_time": null, - "execrecord": 3, - "reused": false, - "start_time": null - }, - "model": "archive.runcomponent", - "pk": 3 - }, - { - "fields": { - "pipelinestep": 1, - "run": 3 - }, - "model": "archive.runstep", - "pk": 1 - }, - { - "fields": { - "pipelinestep": 2, - "run": 4 - }, - "model": "archive.runstep", - "pk": 2 - }, - { - "fields": { - "pipelinestep": 4, - "run": 5 - }, - "model": "archive.runstep", - "pk": 3 - }, - { - "fields": { - "end_time": null, - "invoking_record": 1, - "record": 1, - "start_time": null - }, - "model": "archive.execlog", - "pk": 1 - }, - { - "fields": { - "end_time": null, - "invoking_record": 2, - "record": 2, - "start_time": null - }, - "model": "archive.execlog", - "pk": 2 - }, - { - "fields": { - "end_time": null, - "invoking_record": 3, - "record": 3, - "start_time": null - }, - "model": "archive.execlog", - "pk": 3 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "", - "error_redacted": false, - "execlog": 1, - "install_failed": false, - "output_log": "", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 1 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "", - "error_redacted": false, - "execlog": 2, - "install_failed": false, - "output_log": "", - "output_redacted": false, - "return_code": 1 - }, - "model": "archive.methodoutput", - "pk": 2 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "", - "error_redacted": false, - "execlog": 3, - "install_failed": false, - "output_log": "", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 14, - "dataset": 1, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 2, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 13, - "dataset": 3, - "num_rows": 13 - }, - "model": "librarian.datasetstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 13, - "dataset": 4, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 6, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 5 - }, - { - "fields": { - "compounddatatype": 14, - "dataset": 7, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 6 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 8, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 7 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 9, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 8 - }, - { - "fields": { - "compounddatatype": 13, - "dataset": 10, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 9 - }, - { - "fields": { - "compounddatatype": 14, - "dataset": 13, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 10 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 14, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 11 - }, - { - "fields": { - "compounddatatype": 17, - "dataset": 15, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 12 - }, - { - "fields": { - "compounddatatype": 18, - "dataset": 16, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 13 - }, - { - "fields": { - "compounddatatype": 18, - "dataset": 17, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 14 - }, - { - "fields": { - "generator": 1 - }, - "model": "librarian.execrecord", - "pk": 1 - }, - { - "fields": { - "generator": 2 - }, - "model": "librarian.execrecord", - "pk": 2 - }, - { - "fields": { - "generator": 3 - }, - "model": "librarian.execrecord", - "pk": 3 - }, - { - "fields": { - "dataset": 14, - "execrecord": 1, - "generic_input": 3 - }, - "model": "librarian.execrecordin", - "pk": 1 - }, - { - "fields": { - "dataset": 10, - "execrecord": 1, - "generic_input": 4 - }, - "model": "librarian.execrecordin", - "pk": 2 - }, - { - "fields": { - "dataset": 12, - "execrecord": 2, - "generic_input": 1 - }, - "model": "librarian.execrecordin", - "pk": 3 - }, - { - "fields": { - "dataset": 13, - "execrecord": 3, - "generic_input": 6 - }, - "model": "librarian.execrecordin", - "pk": 4 - }, - { - "fields": { - "dataset": 14, - "execrecord": 3, - "generic_input": 7 - }, - "model": "librarian.execrecordin", - "pk": 5 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 4, - "step_num": 1, - "transformation": 2, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 1 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 1, - "transformation": 1, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 2 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 2, - "transformation": 4, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 3 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 3, - "transformation": 3, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 4 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 1 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 2 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 3 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 4 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 5 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 6 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 7 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 8 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 9 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 10 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 11 - }, - { - "fields": { - "dest": 3, - "keep_output": false, - "pipelinestep": 1, - "source": 11, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 1 - }, - { - "fields": { - "dest": 4, - "keep_output": false, - "pipelinestep": 1, - "source": 12, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 2 - }, - { - "fields": { - "dest": 1, - "keep_output": false, - "pipelinestep": 2, - "source": 15, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 4 - }, - { - "fields": { - "dest": 11, - "keep_output": false, - "pipelinestep": 3, - "source": 13, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 5 - }, - { - "fields": { - "dest": 12, - "keep_output": false, - "pipelinestep": 3, - "source": 14, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 6 - }, - { - "fields": { - "dest": 7, - "keep_output": false, - "pipelinestep": 4, - "source": 2, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 7 - }, - { - "fields": { - "dest": 6, - "keep_output": false, - "pipelinestep": 4, - "source": 16, - "source_step": 2 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 8 - }, - { - "fields": { - "cable": 5, - "dest_pin": 21, - "source_pin": 23 - }, - "model": "pipeline.customcablewire", - "pk": 1 - }, - { - "fields": { - "cable": 5, - "dest_pin": 20, - "source_pin": 25 - }, - "model": "pipeline.customcablewire", - "pk": 2 - }, - { - "fields": { - "cable": 7, - "dest_pin": 21, - "source_pin": 20 - }, - "model": "pipeline.customcablewire", - "pk": 3 - }, - { - "fields": { - "cable": 7, - "dest_pin": 20, - "source_pin": 21 - }, - "model": "pipeline.customcablewire", - "pk": 4 - }, - { - "fields": { - "cable": 9, - "dest_pin": 21, - "source_pin": 24 - }, - "model": "pipeline.customcablewire", - "pk": 5 - }, - { - "fields": { - "cable": 9, - "dest_pin": 20, - "source_pin": 25 - }, - "model": "pipeline.customcablewire", - "pk": 6 - }, - { - "fields": { - "output_cdt": 14, - "output_idx": 1, - "output_name": "D1_out", - "pipeline": 4, - "source": 5, - "source_step": 1 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 3 - }, - { - "fields": { - "output_cdt": 12, - "output_idx": 1, - "output_name": "E1_out", - "pipeline": 5, - "source": 16, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 9 - }, - { - "fields": { - "output_cdt": 13, - "output_idx": 2, - "output_name": "E2_out", - "pipeline": 5, - "source": 8, - "source_step": 3 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 10 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 3, - "output_name": "E3_rawout", - "pipeline": 5, - "source": 10, - "source_step": 3 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 11 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 1 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 2 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 3 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 4 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 5 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 6 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 7 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 8 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 9 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 10 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 11 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 12 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 13 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 14 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 15 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 16 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 17 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 18 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 19 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 20 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 21 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 22 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 23 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 24 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 25 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 2 - }, - "model": "transformation.xputstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 3 - }, - "model": "transformation.xputstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 4 - }, - "model": "transformation.xputstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": 5, - "min_row": null, - "transf_xput": 5 - }, - "model": "transformation.xputstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": null, - "min_row": null, - "transf_xput": 6 - }, - "model": "transformation.xputstructure", - "pk": 5 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 7 - }, - "model": "transformation.xputstructure", - "pk": 6 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 8 - }, - "model": "transformation.xputstructure", - "pk": 7 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 11 - }, - "model": "transformation.xputstructure", - "pk": 8 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 12 - }, - "model": "transformation.xputstructure", - "pk": 9 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": null, - "min_row": null, - "transf_xput": 13 - }, - "model": "transformation.xputstructure", - "pk": 10 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": 10, - "transf_xput": 14 - }, - "model": "transformation.xputstructure", - "pk": 11 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": 5, - "min_row": null, - "transf_xput": 16 - }, - "model": "transformation.xputstructure", - "pk": 12 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": 5, - "min_row": null, - "transf_xput": 17 - }, - "model": "transformation.xputstructure", - "pk": 13 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 18 - }, - "model": "transformation.xputstructure", - "pk": 14 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 20 - }, - "model": "transformation.xputstructure", - "pk": 15 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 21 - }, - "model": "transformation.xputstructure", - "pk": 16 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 22 - }, - "model": "transformation.xputstructure", - "pk": 17 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 23 - }, - "model": "transformation.xputstructure", - "pk": 18 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "A1_rawin", - "transformation": 1 - }, - "model": "transformation.transformationinput", - "pk": 1 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "B1_in", - "transformation": 2 - }, - "model": "transformation.transformationinput", - "pk": 3 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "B2_in", - "transformation": 2 - }, - "model": "transformation.transformationinput", - "pk": 4 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "C1_in", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 6 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "C2_in", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 7 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "D1_in", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 11 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "D2_in", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 12 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "E1_in", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 13 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "E2_in", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 14 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "E3_rawin", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 15 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "strings", - "transformation": 6 - }, - "model": "transformation.transformationinput", - "pk": 20 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "strings", - "transformation": 7 - }, - "model": "transformation.transformationinput", - "pk": 22 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "raw", - "transformation": 8 - }, - "model": "transformation.transformationinput", - "pk": 24 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "A1_out", - "transformation": 1 - }, - "model": "transformation.transformationoutput", - "pk": 2 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "B1_out", - "transformation": 2 - }, - "model": "transformation.transformationoutput", - "pk": 5 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "C1_out", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 8 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "C2_rawout", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 9 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "C3_rawout", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 10 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "D1_out", - "transformation": 4 - }, - "model": "transformation.transformationoutput", - "pk": 16 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "E1_out", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 17 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "E2_out", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 18 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "E3_rawout", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 19 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "same_strings", - "transformation": 6 - }, - "model": "transformation.transformationoutput", - "pk": 21 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "untouched_strings", - "transformation": 7 - }, - "model": "transformation.transformationoutput", - "pk": 23 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "same_raw", - "transformation": 8 - }, - "model": "transformation.transformationoutput", - "pk": 25 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:00Z", - "email": "lennon@thebeatles.com", - "first_name": "", - "groups": [ - 1 - ], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$KZolXWnv6wks$+LmPJmmsiJ/0rSbRze1S0aXC9Gs+X+NUh8LfMmnG4tM=", - "user_permissions": [], - "username": "john" - }, - "model": "auth.user", - "pk": 2 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:00Z", - "email": "starr@thebeatles.com", - "first_name": "", - "groups": [], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$OZFN2YW6hCyV$4j+wtoGKxTL9MOdK9ZRDUzmEhVzvXTLWWvjUohh/6WU=", - "user_permissions": [], - "username": "ringo" - }, - "model": "auth.user", - "pk": 3 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:02Z", - "email": "bob@talabs.com", - "first_name": "", - "groups": [ - 1 - ], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$EdQSsQ3Xqxqs$HaWX8nPr9TURKgnyc8kr+Ve934czaHjb4GkGd3fzonM=", - "user_permissions": [], - "username": "bob" - }, - "model": "auth.user", - "pk": 4 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:00Z", - "description": "String consisting of ACGTacgt", - "groups_allowed": [ - 1 - ], - "name": "DNANucSeq", - "prototype": null, - "restricts": [ - 1 - ], - "user": 2, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 8 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:00Z", - "description": "String consisting of ACGUacgu", - "groups_allowed": [ - 1 - ], - "name": "RNANucSeq", - "prototype": null, - "restricts": [ - 1 - ], - "user": 2, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 9 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:02Z", - "description": "sequences of ASCII characters", - "groups_allowed": [ - 1 - ], - "name": "my_string", - "prototype": null, - "restricts": [ - 1 - ], - "user": 4, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 10 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [], - "name": "", - "user": 2, - "users_allowed": [ - 3 - ] - }, - "model": "metadata.compounddatatype", - "pk": 6 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 7 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 8 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 9 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 10 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 11 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 12 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 13 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 14 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 15 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 16 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 17 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 18 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 4, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 19 - }, - { - "fields": { - "_runstate": 1, - "description": "", - "end_time": null, - "groups_allowed": [ - 1 - ], - "name": "pD_run", - "parent_runstep": null, - "paused_by": null, - "pipeline": 4, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "", - "start_time": null, - "stopped_by": null, - "time_queued": "2017-04-25T00:52:01.635Z", - "user": 2, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 1 - }, - { - "fields": { - "_runstate": 1, - "description": "", - "end_time": null, - "groups_allowed": [ - 1 - ], - "name": "pE_run", - "parent_runstep": null, - "paused_by": null, - "pipeline": 5, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "", - "start_time": null, - "stopped_by": null, - "time_queued": "2017-04-25T00:52:01.644Z", - "user": 2, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 2 - }, - { - "fields": { - "_runstate": 1, - "description": "", - "end_time": null, - "groups_allowed": [], - "name": "", - "parent_runstep": null, - "paused_by": null, - "pipeline": 4, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "", - "start_time": null, - "stopped_by": null, - "time_queued": "2017-04-25T00:52:01.663Z", - "user": 2, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 3 - }, - { - "fields": { - "_runstate": 1, - "description": "", - "end_time": null, - "groups_allowed": [], - "name": "", - "parent_runstep": null, - "paused_by": null, - "pipeline": 5, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "", - "start_time": null, - "stopped_by": null, - "time_queued": "2017-04-25T00:52:01.752Z", - "user": 2, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 4 - }, - { - "fields": { - "_runstate": 1, - "description": "", - "end_time": null, - "groups_allowed": [], - "name": "", - "parent_runstep": null, - "paused_by": null, - "pipeline": 5, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "", - "start_time": null, - "stopped_by": null, - "time_queued": "2017-04-25T00:52:01.820Z", - "user": 2, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 5 - }, - { - "fields": { - "MD5_checksum": "ab7afe2f453d27fd168641a1f9271f26", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_triplet.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:00.602Z", - "name": "triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "02d5b66858d003b81ae3dc712b74e85c", - "_redacted": false, - "dataset_file": "Datasets/2017_04/doublet_cdt.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:00.671Z", - "name": "doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "29feb269edcac613d6eea432c14640bd", - "_redacted": false, - "dataset_file": "Datasets/2017_04/singlet_cdt_large.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:00.729Z", - "name": "singlet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 3 - }, - { - "fields": { - "MD5_checksum": "00fd50bfe7ac63e2165a442f4101e05c", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_singlet.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:00.781Z", - "name": "singlet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 4 - }, - { - "fields": { - "MD5_checksum": "7dc85e11b5c02e434af5bd3b3da9938e", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_raw.fasta", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:00.822Z", - "name": "raw_DS", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 5 - }, - { - "fields": { - "MD5_checksum": "542676b23e121d16db8d41ccdae65fd1", - "_redacted": false, - "dataset_file": "", - "date_created": "2000-01-01T00:00:01Z", - "description": "", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:00.857Z", - "name": "D1_in_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 6 - }, - { - "fields": { - "MD5_checksum": "56f9e5585b9d699829e2fda931d944f8", - "_redacted": false, - "dataset_file": "Datasets/2017_04/C1_in_triplet.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "triplet 3 rows", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:00.907Z", - "name": "C1_in_triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 7 - }, - { - "fields": { - "MD5_checksum": "10a4bd15d1300fcaf5a3bdad54ef1d28", - "_redacted": false, - "dataset_file": "", - "date_created": "2000-01-01T00:00:01Z", - "description": "", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:00.973Z", - "name": "C2_in_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 8 - }, - { - "fields": { - "MD5_checksum": "10a4bd15d1300fcaf5a3bdad54ef1d28", - "_redacted": false, - "dataset_file": "Datasets/2017_04/E11_32_output.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "result of E11_32 fed by doublet_cdt.csv", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:01.025Z", - "name": "E11_32 output doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 9 - }, - { - "fields": { - "MD5_checksum": "00fd50bfe7ac63e2165a442f4101e05c", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_singlet_bEFM8wM.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:01.084Z", - "name": "raw", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 10 - }, - { - "fields": { - "MD5_checksum": "7dc85e11b5c02e434af5bd3b3da9938e", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_raw_ylxf5vU.fasta", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:01.126Z", - "name": "C2_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 11 - }, - { - "fields": { - "MD5_checksum": "7dc85e11b5c02e434af5bd3b3da9938e", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_raw_HSYJtoM.fasta", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:01.155Z", - "name": "C3_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 12 - }, - { - "fields": { - "MD5_checksum": "56f9e5585b9d699829e2fda931d944f8", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_triplet_3_rows.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:01.187Z", - "name": "triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 13 - }, - { - "fields": { - "MD5_checksum": "83957f8aea1c75da9f7fbf9bc90da979", - "_redacted": false, - "dataset_file": "Datasets/2017_04/doublet_remuxed_from_t3r.csv", - "date_created": "2000-01-01T00:00:02Z", - "description": "doublet remuxed from triplet", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:01.253Z", - "name": "E1_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 14 - }, - { - "fields": { - "MD5_checksum": "d83866e4c23cd9503aa2f9fc565502b5", - "_redacted": false, - "dataset_file": "Datasets/2017_04/DNA_triplet.csv", - "date_created": "2000-01-01T00:00:02Z", - "description": "DNA triplet data", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:01.312Z", - "name": "DNA_triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 15 - }, - { - "fields": { - "MD5_checksum": "9eab0dfed5fcc89bd581f61995816ef0", - "_redacted": false, - "dataset_file": "Datasets/2017_04/E01_21_DNA_doublet.csv", - "date_created": "2000-01-01T00:00:02Z", - "description": "DNA doublet data coming from DNA_triplet.csv but remultiplexed according to cable E01_21", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:01.449Z", - "name": "E01_21_DNA_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 16 - }, - { - "fields": { - "MD5_checksum": "9b55b9431da6dba74ea61edf8e2ac044", - "_redacted": false, - "dataset_file": "Datasets/2017_04/E21_41_DNA_doublet.csv", - "date_created": "2000-01-01T00:00:02Z", - "description": "DNA doublet data coming from DNA_triplet.csv but remultiplexed according to cable E21_41", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:01.550Z", - "name": "E21_41_DNA_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 17 - }, - { - "fields": { - "description": "Just a CR", - "filename": "generic_script.py", - "groups_allowed": [ - 1 - ], - "name": "genericCR", - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 1 - }, - { - "fields": { - "description": "a script to do nothing", - "filename": "noop.sh", - "groups_allowed": [ - 1 - ], - "name": "noop", - "user": 4, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "e353a62fe8cb6a56d31b8c65d70b217d", - "coderesource": 1, - "content_file": "CodeResources/generic_script.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "desc", - "revision_name": "v1", - "revision_number": 1, - "revision_parent": null, - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "d218c00725d473639f347d456de381d8", - "coderesource": 2, - "content_file": "CodeResources/fdopen", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "1", - "revision_number": 1, - "revision_parent": null, - "user": 4, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 2 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 1 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 2, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 2 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 3, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 3 - }, - { - "fields": { - "driver": 2, - "family": 2, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 6 - }, - { - "fields": { - "driver": 2, - "family": 3, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 7 - }, - { - "fields": { - "driver": 2, - "family": 4, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 8 - }, - { - "fields": { - "description": "Holds methods A/B/C", - "groups_allowed": [ - 1 - ], - "name": "method_family", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 1 - }, - { - "fields": { - "description": "a method to do nothing to strings", - "groups_allowed": [ - 1 - ], - "name": "string noop", - "user": 4, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 2 - }, - { - "fields": { - "description": "a TOTALLY DIFFERENT method that TOTALLY does SOMETHING to strings by leaving them alone", - "groups_allowed": [ - 1 - ], - "name": "string trivial", - "user": 4, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 3 - }, - { - "fields": { - "description": "do nothing to raw data", - "groups_allowed": [ - 1 - ], - "name": "raw noop", - "user": 4, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 4 - }, - { - "fields": { - "description": "PF desc", - "groups_allowed": [ - 1 - ], - "name": "Pipeline_family", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 1 - }, - { - "fields": { - "family": 1, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 4 - }, - { - "fields": { - "family": 1, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 2, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "A_desc", - "revision_name": "mA_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 1 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "B_desc", - "revision_name": "mB_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 2 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "C_desc", - "revision_name": "mC_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 3 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "D", - "revision_name": "pD_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 4 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "E", - "revision_name": "pE_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 4, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 6 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 4, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 7 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:03Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 4, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 8 - }, - { - "fields": { - "dataset": 1, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 3, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 3 - }, - { - "fields": { - "dataset": 4, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 4 - }, - { - "fields": { - "dataset": 6, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 5 - }, - { - "fields": { - "dataset": 7, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 6 - }, - { - "fields": { - "dataset": 8, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 7 - }, - { - "fields": { - "dataset": 9, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 8 - }, - { - "fields": { - "dataset": 10, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 9 - }, - { - "fields": { - "dataset": 13, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 10 - }, - { - "fields": { - "dataset": 14, - "end_time": "2000-01-01T00:00:02Z", - "execlog": null, - "start_time": "2000-01-01T00:00:02Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 11 - }, - { - "fields": { - "dataset": 15, - "end_time": "2000-01-01T00:00:02Z", - "execlog": null, - "start_time": "2000-01-01T00:00:02Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 12 - }, - { - "fields": { - "dataset": 16, - "end_time": "2000-01-01T00:00:02Z", - "execlog": null, - "start_time": "2000-01-01T00:00:02Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 13 - }, - { - "fields": { - "dataset": 17, - "end_time": "2000-01-01T00:00:02Z", - "execlog": null, - "start_time": "2000-01-01T00:00:02Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 14 - } -] \ No newline at end of file diff --git a/kive/portal/fixtures/container_run.json b/kive/portal/fixtures/container_run.json index 4210b534e..443a7265c 100644 --- a/kive/portal/fixtures/container_run.json +++ b/kive/portal/fixtures/container_run.json @@ -35,7 +35,7 @@ { "fields": { "argument": 1, - "created": "2019-02-22T00:14:26.864Z", + "created": "2019-06-14T22:38:36.119Z", "dataset": 1, "name": "", "run": 1 @@ -57,14 +57,14 @@ }, { "fields": { - "created": "2019-02-22T00:14:26.778Z", + "created": "2019-06-14T22:38:36.058Z", "description": "", "family": 1, "file": "Containers/kive-default.simg", "file_size": null, "file_type": "SIMG", "groups_allowed": [], - "md5": "a9fd7df68e3a75f121206d4f0f29be65", + "md5": "a9fd7df68e3a75f121206d4f0f29be65", "parent": null, "tag": "vFixture", "user": 1, @@ -81,11 +81,13 @@ "end_time": null, "groups_allowed": [], "is_redacted": false, + "md5": "", "name": "fixture run", + "original_run": null, "priority": 0, "return_code": null, "sandbox_path": "", - "sandbox_size": null, + "sandbox_size": null, "slurm_job_id": null, "start_time": null, "state": "N", @@ -101,16 +103,16 @@ "fields": { "MD5_checksum": "06f7204f2679744fd76e6b111dc506ba", "_redacted": false, - "dataset_file": "Datasets/2019_02/example_names.csv", + "dataset_file": "Datasets/2019_06/example_names.csv", "dataset_size": null, "date_created": "2000-01-01T00:00:00Z", "description": "", "external_path": "", "externalfiledirectory": null, - "file_source": null, "groups_allowed": [], "is_external_missing": false, - "last_time_checked": "2019-02-22T00:14:26.794Z", + "is_uploaded": false, + "last_time_checked": "2019-06-14T22:38:36.066Z", "name": "names.csv", "user": 1, "users_allowed": [] diff --git a/kive/portal/fixtures/deep_nested_run.json b/kive/portal/fixtures/deep_nested_run.json deleted file mode 100644 index 3ec8be146..000000000 --- a/kive/portal/fixtures/deep_nested_run.json +++ /dev/null @@ -1,4780 +0,0 @@ -[ - { - "fields": { - "datatype": 8, - "rule": "^[ACGTacgt]*$", - "ruletype": "regexp" - }, - "model": "metadata.basicconstraint", - "pk": 6 - }, - { - "fields": { - "datatype": 9, - "rule": "^[ACGUacgu]*$", - "ruletype": "regexp" - }, - "model": "metadata.basicconstraint", - "pk": 7 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "label", - "compounddatatype": 5, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 7 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "integer", - "compounddatatype": 5, - "datatype": 4 - }, - "model": "metadata.compounddatatypemember", - "pk": 8 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "float", - "compounddatatype": 5, - "datatype": 3 - }, - "model": "metadata.compounddatatypemember", - "pk": 9 - }, - { - "fields": { - "blankable": false, - "column_idx": 4, - "column_name": "bool", - "compounddatatype": 5, - "datatype": 2 - }, - "model": "metadata.compounddatatypemember", - "pk": 10 - }, - { - "fields": { - "blankable": false, - "column_idx": 5, - "column_name": "rna", - "compounddatatype": 5, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 11 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "label", - "compounddatatype": 6, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 12 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "label", - "compounddatatype": 7, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 13 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "PBMCseq", - "compounddatatype": 7, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 14 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "PLAseq", - "compounddatatype": 7, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 15 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "SeqToComplement", - "compounddatatype": 8, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 16 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "ComplementedSeq", - "compounddatatype": 9, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 17 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "SeqToComplement", - "compounddatatype": 10, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 18 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "ComplementedSeq", - "compounddatatype": 11, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 19 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "x", - "compounddatatype": 12, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 20 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "y", - "compounddatatype": 12, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 21 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "k", - "compounddatatype": 13, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 22 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "a", - "compounddatatype": 14, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 23 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "b", - "compounddatatype": 14, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 24 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "c", - "compounddatatype": 14, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 25 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "a^2", - "compounddatatype": 15, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 26 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "b^2", - "compounddatatype": 15, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 27 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "c^2", - "compounddatatype": 15, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 28 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "StrCol1", - "compounddatatype": 16, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 29 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "DNACol2", - "compounddatatype": 16, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 30 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "StrCol3", - "compounddatatype": 16, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 31 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "a", - "compounddatatype": 17, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 32 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "b", - "compounddatatype": 17, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 33 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "c", - "compounddatatype": 17, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 34 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "x", - "compounddatatype": 18, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 35 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "y", - "compounddatatype": 18, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 36 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "word", - "compounddatatype": 19, - "datatype": 10 - }, - "model": "metadata.compounddatatypemember", - "pk": 37 - }, - { - "fields": { - "dataset": 18, - "index": 1, - "run": 1 - }, - "model": "archive.runinput", - "pk": 1 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:43Z", - "execrecord": null, - "reused": null, - "start_time": "2000-01-01T00:00:06Z" - }, - "model": "archive.runcomponent", - "pk": 1 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:41Z", - "execrecord": null, - "reused": null, - "start_time": "2000-01-01T00:00:07Z" - }, - "model": "archive.runcomponent", - "pk": 2 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:20Z", - "execrecord": 2, - "reused": false, - "start_time": "2000-01-01T00:00:07Z" - }, - "model": "archive.runcomponent", - "pk": 3 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:35Z", - "execrecord": 4, - "reused": false, - "start_time": "2000-01-01T00:00:23Z" - }, - "model": "archive.runcomponent", - "pk": 4 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:42Z", - "execrecord": null, - "reused": null, - "start_time": "2000-01-01T00:00:41Z" - }, - "model": "archive.runcomponent", - "pk": 5 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:42Z", - "execrecord": 2, - "reused": true, - "start_time": "2000-01-01T00:00:42Z" - }, - "model": "archive.runcomponent", - "pk": 6 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:42Z", - "execrecord": 4, - "reused": true, - "start_time": "2000-01-01T00:00:42Z" - }, - "model": "archive.runcomponent", - "pk": 7 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:45Z", - "execrecord": null, - "reused": null, - "start_time": "2000-01-01T00:00:43Z" - }, - "model": "archive.runcomponent", - "pk": 8 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:44Z", - "execrecord": null, - "reused": null, - "start_time": "2000-01-01T00:00:43Z" - }, - "model": "archive.runcomponent", - "pk": 9 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:43Z", - "execrecord": 2, - "reused": true, - "start_time": "2000-01-01T00:00:43Z" - }, - "model": "archive.runcomponent", - "pk": 10 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:44Z", - "execrecord": 4, - "reused": true, - "start_time": "2000-01-01T00:00:43Z" - }, - "model": "archive.runcomponent", - "pk": 11 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:45Z", - "execrecord": null, - "reused": null, - "start_time": "2000-01-01T00:00:44Z" - }, - "model": "archive.runcomponent", - "pk": 12 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:44Z", - "execrecord": 2, - "reused": true, - "start_time": "2000-01-01T00:00:44Z" - }, - "model": "archive.runcomponent", - "pk": 13 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:45Z", - "execrecord": 4, - "reused": true, - "start_time": "2000-01-01T00:00:44Z" - }, - "model": "archive.runcomponent", - "pk": 14 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:47Z", - "execrecord": null, - "reused": null, - "start_time": "2000-01-01T00:00:45Z" - }, - "model": "archive.runcomponent", - "pk": 15 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:46Z", - "execrecord": null, - "reused": null, - "start_time": "2000-01-01T00:00:45Z" - }, - "model": "archive.runcomponent", - "pk": 16 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:46Z", - "execrecord": 2, - "reused": true, - "start_time": "2000-01-01T00:00:45Z" - }, - "model": "archive.runcomponent", - "pk": 17 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:46Z", - "execrecord": 4, - "reused": true, - "start_time": "2000-01-01T00:00:46Z" - }, - "model": "archive.runcomponent", - "pk": 18 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:47Z", - "execrecord": null, - "reused": null, - "start_time": "2000-01-01T00:00:46Z" - }, - "model": "archive.runcomponent", - "pk": 19 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:47Z", - "execrecord": 2, - "reused": true, - "start_time": "2000-01-01T00:00:47Z" - }, - "model": "archive.runcomponent", - "pk": 20 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:47Z", - "execrecord": 4, - "reused": true, - "start_time": "2000-01-01T00:00:47Z" - }, - "model": "archive.runcomponent", - "pk": 21 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:06Z", - "execrecord": 1, - "reused": false, - "start_time": "2000-01-01T00:00:06Z" - }, - "model": "archive.runcomponent", - "pk": 22 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:07Z", - "execrecord": 1, - "reused": true, - "start_time": "2000-01-01T00:00:07Z" - }, - "model": "archive.runcomponent", - "pk": 23 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:08Z", - "execrecord": 1, - "reused": false, - "start_time": "2000-01-01T00:00:07Z" - }, - "model": "archive.runcomponent", - "pk": 24 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:23Z", - "execrecord": 3, - "reused": false, - "start_time": "2000-01-01T00:00:23Z" - }, - "model": "archive.runcomponent", - "pk": 25 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:39Z", - "execrecord": 5, - "reused": false, - "start_time": "2000-01-01T00:00:38Z" - }, - "model": "archive.runcomponent", - "pk": 26 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:42Z", - "execrecord": 5, - "reused": true, - "start_time": "2000-01-01T00:00:41Z" - }, - "model": "archive.runcomponent", - "pk": 27 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:42Z", - "execrecord": 5, - "reused": true, - "start_time": "2000-01-01T00:00:42Z" - }, - "model": "archive.runcomponent", - "pk": 28 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:42Z", - "execrecord": 3, - "reused": true, - "start_time": "2000-01-01T00:00:42Z" - }, - "model": "archive.runcomponent", - "pk": 29 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:42Z", - "execrecord": 5, - "reused": true, - "start_time": "2000-01-01T00:00:42Z" - }, - "model": "archive.runcomponent", - "pk": 30 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:43Z", - "execrecord": 5, - "reused": true, - "start_time": "2000-01-01T00:00:42Z" - }, - "model": "archive.runcomponent", - "pk": 31 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:43Z", - "execrecord": 5, - "reused": true, - "start_time": "2000-01-01T00:00:43Z" - }, - "model": "archive.runcomponent", - "pk": 32 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:43Z", - "execrecord": 5, - "reused": true, - "start_time": "2000-01-01T00:00:43Z" - }, - "model": "archive.runcomponent", - "pk": 33 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:43Z", - "execrecord": 5, - "reused": true, - "start_time": "2000-01-01T00:00:43Z" - }, - "model": "archive.runcomponent", - "pk": 34 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:44Z", - "execrecord": 3, - "reused": true, - "start_time": "2000-01-01T00:00:43Z" - }, - "model": "archive.runcomponent", - "pk": 35 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:44Z", - "execrecord": 5, - "reused": true, - "start_time": "2000-01-01T00:00:44Z" - }, - "model": "archive.runcomponent", - "pk": 36 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:44Z", - "execrecord": 5, - "reused": true, - "start_time": "2000-01-01T00:00:44Z" - }, - "model": "archive.runcomponent", - "pk": 37 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:44Z", - "execrecord": 5, - "reused": true, - "start_time": "2000-01-01T00:00:44Z" - }, - "model": "archive.runcomponent", - "pk": 38 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:45Z", - "execrecord": 3, - "reused": true, - "start_time": "2000-01-01T00:00:44Z" - }, - "model": "archive.runcomponent", - "pk": 39 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:45Z", - "execrecord": 5, - "reused": true, - "start_time": "2000-01-01T00:00:45Z" - }, - "model": "archive.runcomponent", - "pk": 40 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:45Z", - "execrecord": 5, - "reused": true, - "start_time": "2000-01-01T00:00:45Z" - }, - "model": "archive.runcomponent", - "pk": 41 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:45Z", - "execrecord": 5, - "reused": true, - "start_time": "2000-01-01T00:00:45Z" - }, - "model": "archive.runcomponent", - "pk": 42 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:45Z", - "execrecord": 5, - "reused": true, - "start_time": "2000-01-01T00:00:45Z" - }, - "model": "archive.runcomponent", - "pk": 43 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:46Z", - "execrecord": 5, - "reused": true, - "start_time": "2000-01-01T00:00:46Z" - }, - "model": "archive.runcomponent", - "pk": 44 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:46Z", - "execrecord": 3, - "reused": true, - "start_time": "2000-01-01T00:00:46Z" - }, - "model": "archive.runcomponent", - "pk": 45 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:46Z", - "execrecord": 5, - "reused": true, - "start_time": "2000-01-01T00:00:46Z" - }, - "model": "archive.runcomponent", - "pk": 46 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:46Z", - "execrecord": 5, - "reused": true, - "start_time": "2000-01-01T00:00:46Z" - }, - "model": "archive.runcomponent", - "pk": 47 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:47Z", - "execrecord": 5, - "reused": true, - "start_time": "2000-01-01T00:00:47Z" - }, - "model": "archive.runcomponent", - "pk": 48 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:47Z", - "execrecord": 3, - "reused": true, - "start_time": "2000-01-01T00:00:47Z" - }, - "model": "archive.runcomponent", - "pk": 49 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:47Z", - "execrecord": 5, - "reused": true, - "start_time": "2000-01-01T00:00:47Z" - }, - "model": "archive.runcomponent", - "pk": 50 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:47Z", - "execrecord": 5, - "reused": true, - "start_time": "2000-01-01T00:00:47Z" - }, - "model": "archive.runcomponent", - "pk": 51 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:48Z", - "execrecord": 5, - "reused": true, - "start_time": "2000-01-01T00:00:47Z" - }, - "model": "archive.runcomponent", - "pk": 52 - }, - { - "fields": { - "pipelinestep": 9, - "run": 1 - }, - "model": "archive.runstep", - "pk": 1 - }, - { - "fields": { - "pipelinestep": 7, - "run": 2 - }, - "model": "archive.runstep", - "pk": 2 - }, - { - "fields": { - "pipelinestep": 5, - "run": 3 - }, - "model": "archive.runstep", - "pk": 3 - }, - { - "fields": { - "pipelinestep": 6, - "run": 3 - }, - "model": "archive.runstep", - "pk": 4 - }, - { - "fields": { - "pipelinestep": 8, - "run": 2 - }, - "model": "archive.runstep", - "pk": 5 - }, - { - "fields": { - "pipelinestep": 5, - "run": 4 - }, - "model": "archive.runstep", - "pk": 6 - }, - { - "fields": { - "pipelinestep": 6, - "run": 4 - }, - "model": "archive.runstep", - "pk": 7 - }, - { - "fields": { - "pipelinestep": 10, - "run": 1 - }, - "model": "archive.runstep", - "pk": 8 - }, - { - "fields": { - "pipelinestep": 7, - "run": 5 - }, - "model": "archive.runstep", - "pk": 9 - }, - { - "fields": { - "pipelinestep": 5, - "run": 6 - }, - "model": "archive.runstep", - "pk": 10 - }, - { - "fields": { - "pipelinestep": 6, - "run": 6 - }, - "model": "archive.runstep", - "pk": 11 - }, - { - "fields": { - "pipelinestep": 8, - "run": 5 - }, - "model": "archive.runstep", - "pk": 12 - }, - { - "fields": { - "pipelinestep": 5, - "run": 7 - }, - "model": "archive.runstep", - "pk": 13 - }, - { - "fields": { - "pipelinestep": 6, - "run": 7 - }, - "model": "archive.runstep", - "pk": 14 - }, - { - "fields": { - "pipelinestep": 11, - "run": 1 - }, - "model": "archive.runstep", - "pk": 15 - }, - { - "fields": { - "pipelinestep": 7, - "run": 8 - }, - "model": "archive.runstep", - "pk": 16 - }, - { - "fields": { - "pipelinestep": 5, - "run": 9 - }, - "model": "archive.runstep", - "pk": 17 - }, - { - "fields": { - "pipelinestep": 6, - "run": 9 - }, - "model": "archive.runstep", - "pk": 18 - }, - { - "fields": { - "pipelinestep": 8, - "run": 8 - }, - "model": "archive.runstep", - "pk": 19 - }, - { - "fields": { - "pipelinestep": 5, - "run": 10 - }, - "model": "archive.runstep", - "pk": 20 - }, - { - "fields": { - "pipelinestep": 6, - "run": 10 - }, - "model": "archive.runstep", - "pk": 21 - }, - { - "fields": { - "PSIC": 18, - "dest_runstep": 1 - }, - "model": "archive.runsic", - "pk": 22 - }, - { - "fields": { - "PSIC": 15, - "dest_runstep": 2 - }, - "model": "archive.runsic", - "pk": 23 - }, - { - "fields": { - "PSIC": 12, - "dest_runstep": 3 - }, - "model": "archive.runsic", - "pk": 24 - }, - { - "fields": { - "PSIC": 13, - "dest_runstep": 4 - }, - "model": "archive.runsic", - "pk": 25 - }, - { - "fields": { - "PSIC": 16, - "dest_runstep": 5 - }, - "model": "archive.runsic", - "pk": 27 - }, - { - "fields": { - "PSIC": 12, - "dest_runstep": 6 - }, - "model": "archive.runsic", - "pk": 28 - }, - { - "fields": { - "PSIC": 13, - "dest_runstep": 7 - }, - "model": "archive.runsic", - "pk": 29 - }, - { - "fields": { - "PSIC": 19, - "dest_runstep": 8 - }, - "model": "archive.runsic", - "pk": 32 - }, - { - "fields": { - "PSIC": 15, - "dest_runstep": 9 - }, - "model": "archive.runsic", - "pk": 33 - }, - { - "fields": { - "PSIC": 12, - "dest_runstep": 10 - }, - "model": "archive.runsic", - "pk": 34 - }, - { - "fields": { - "PSIC": 13, - "dest_runstep": 11 - }, - "model": "archive.runsic", - "pk": 35 - }, - { - "fields": { - "PSIC": 16, - "dest_runstep": 12 - }, - "model": "archive.runsic", - "pk": 37 - }, - { - "fields": { - "PSIC": 12, - "dest_runstep": 13 - }, - "model": "archive.runsic", - "pk": 38 - }, - { - "fields": { - "PSIC": 13, - "dest_runstep": 14 - }, - "model": "archive.runsic", - "pk": 39 - }, - { - "fields": { - "PSIC": 20, - "dest_runstep": 15 - }, - "model": "archive.runsic", - "pk": 42 - }, - { - "fields": { - "PSIC": 15, - "dest_runstep": 16 - }, - "model": "archive.runsic", - "pk": 43 - }, - { - "fields": { - "PSIC": 12, - "dest_runstep": 17 - }, - "model": "archive.runsic", - "pk": 44 - }, - { - "fields": { - "PSIC": 13, - "dest_runstep": 18 - }, - "model": "archive.runsic", - "pk": 45 - }, - { - "fields": { - "PSIC": 16, - "dest_runstep": 19 - }, - "model": "archive.runsic", - "pk": 47 - }, - { - "fields": { - "PSIC": 12, - "dest_runstep": 20 - }, - "model": "archive.runsic", - "pk": 48 - }, - { - "fields": { - "PSIC": 13, - "dest_runstep": 21 - }, - "model": "archive.runsic", - "pk": 49 - }, - { - "fields": { - "pipelineoutputcable": 14, - "run": 3 - }, - "model": "archive.runoutputcable", - "pk": 26 - }, - { - "fields": { - "pipelineoutputcable": 14, - "run": 4 - }, - "model": "archive.runoutputcable", - "pk": 30 - }, - { - "fields": { - "pipelineoutputcable": 17, - "run": 2 - }, - "model": "archive.runoutputcable", - "pk": 31 - }, - { - "fields": { - "pipelineoutputcable": 14, - "run": 6 - }, - "model": "archive.runoutputcable", - "pk": 36 - }, - { - "fields": { - "pipelineoutputcable": 14, - "run": 7 - }, - "model": "archive.runoutputcable", - "pk": 40 - }, - { - "fields": { - "pipelineoutputcable": 17, - "run": 5 - }, - "model": "archive.runoutputcable", - "pk": 41 - }, - { - "fields": { - "pipelineoutputcable": 14, - "run": 9 - }, - "model": "archive.runoutputcable", - "pk": 46 - }, - { - "fields": { - "pipelineoutputcable": 14, - "run": 10 - }, - "model": "archive.runoutputcable", - "pk": 50 - }, - { - "fields": { - "pipelineoutputcable": 17, - "run": 8 - }, - "model": "archive.runoutputcable", - "pk": 51 - }, - { - "fields": { - "pipelineoutputcable": 21, - "run": 1 - }, - "model": "archive.runoutputcable", - "pk": 52 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:06Z", - "invoking_record": 22, - "record": 22, - "start_time": "2000-01-01T00:00:06Z" - }, - "model": "archive.execlog", - "pk": 1 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:08Z", - "invoking_record": 24, - "record": 24, - "start_time": "2000-01-01T00:00:08Z" - }, - "model": "archive.execlog", - "pk": 2 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:08Z", - "invoking_record": 3, - "record": 3, - "start_time": "2000-01-01T00:00:08Z" - }, - "model": "archive.execlog", - "pk": 3 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:23Z", - "invoking_record": 25, - "record": 25, - "start_time": "2000-01-01T00:00:23Z" - }, - "model": "archive.execlog", - "pk": 4 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:24Z", - "invoking_record": 4, - "record": 4, - "start_time": "2000-01-01T00:00:24Z" - }, - "model": "archive.execlog", - "pk": 5 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:38Z", - "invoking_record": 26, - "record": 26, - "start_time": "2000-01-01T00:00:38Z" - }, - "model": "archive.execlog", - "pk": 6 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step1_stderr_slurmIDJ_nodeN.txt", - "error_redacted": false, - "execlog": 3, - "install_failed": false, - "output_log": "Logs/step1_stdout_slurmIDJ_nodeN.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 1 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step2_stderr_slurmIDJ_nodeN.txt", - "error_redacted": false, - "execlog": 5, - "install_failed": false, - "output_log": "Logs/step2_stdout_slurmIDJ_nodeN.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 14, - "dataset": 1, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 2, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 13, - "dataset": 3, - "num_rows": 13 - }, - "model": "librarian.datasetstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 13, - "dataset": 4, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 6, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 5 - }, - { - "fields": { - "compounddatatype": 14, - "dataset": 7, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 6 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 8, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 7 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 9, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 8 - }, - { - "fields": { - "compounddatatype": 13, - "dataset": 10, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 9 - }, - { - "fields": { - "compounddatatype": 14, - "dataset": 13, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 10 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 14, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 11 - }, - { - "fields": { - "compounddatatype": 17, - "dataset": 15, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 12 - }, - { - "fields": { - "compounddatatype": 18, - "dataset": 16, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 13 - }, - { - "fields": { - "compounddatatype": 18, - "dataset": 17, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 14 - }, - { - "fields": { - "compounddatatype": 19, - "dataset": 18, - "num_rows": 1 - }, - "model": "librarian.datasetstructure", - "pk": 15 - }, - { - "fields": { - "compounddatatype": 19, - "dataset": 19, - "num_rows": 1 - }, - "model": "librarian.datasetstructure", - "pk": 16 - }, - { - "fields": { - "compounddatatype": 19, - "dataset": 20, - "num_rows": 1 - }, - "model": "librarian.datasetstructure", - "pk": 17 - }, - { - "fields": { - "generator": 1 - }, - "model": "librarian.execrecord", - "pk": 1 - }, - { - "fields": { - "generator": 3 - }, - "model": "librarian.execrecord", - "pk": 2 - }, - { - "fields": { - "generator": 4 - }, - "model": "librarian.execrecord", - "pk": 3 - }, - { - "fields": { - "generator": 5 - }, - "model": "librarian.execrecord", - "pk": 4 - }, - { - "fields": { - "generator": 6 - }, - "model": "librarian.execrecord", - "pk": 5 - }, - { - "fields": { - "dataset": 18, - "execrecord": 1, - "generic_input": 32 - }, - "model": "librarian.execrecordin", - "pk": 1 - }, - { - "fields": { - "dataset": 18, - "execrecord": 2, - "generic_input": 20 - }, - "model": "librarian.execrecordin", - "pk": 2 - }, - { - "fields": { - "dataset": 19, - "execrecord": 3, - "generic_input": 21 - }, - "model": "librarian.execrecordin", - "pk": 3 - }, - { - "fields": { - "dataset": 19, - "execrecord": 4, - "generic_input": 20 - }, - "model": "librarian.execrecordin", - "pk": 4 - }, - { - "fields": { - "dataset": 20, - "execrecord": 5, - "generic_input": 21 - }, - "model": "librarian.execrecordin", - "pk": 5 - }, - { - "fields": { - "dataset": 18, - "execrecord": 1, - "generic_output": 29 - }, - "model": "librarian.execrecordout", - "pk": 1 - }, - { - "fields": { - "dataset": 19, - "execrecord": 2, - "generic_output": 21 - }, - "model": "librarian.execrecordout", - "pk": 2 - }, - { - "fields": { - "dataset": 19, - "execrecord": 3, - "generic_output": 20 - }, - "model": "librarian.execrecordout", - "pk": 3 - }, - { - "fields": { - "dataset": 20, - "execrecord": 4, - "generic_output": 21 - }, - "model": "librarian.execrecordout", - "pk": 4 - }, - { - "fields": { - "dataset": 20, - "execrecord": 5, - "generic_output": 28 - }, - "model": "librarian.execrecordout", - "pk": 5 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 4, - "step_num": 1, - "transformation": 2, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 1 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 1, - "transformation": 1, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 2 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 2, - "transformation": 4, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 3 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 3, - "transformation": 3, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 4 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 9, - "step_num": 1, - "transformation": 6, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 5 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 9, - "step_num": 2, - "transformation": 6, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 6 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 10, - "step_num": 1, - "transformation": 9, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 7 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 10, - "step_num": 2, - "transformation": 9, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 8 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 11, - "step_num": 1, - "transformation": 10, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 9 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 11, - "step_num": 2, - "transformation": 10, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 10 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 11, - "step_num": 3, - "transformation": 10, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 11 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 1 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 2 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 3 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 4 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 5 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 6 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 7 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 8 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 9 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 10 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 11 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 12 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 13 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 14 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 15 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 16 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 17 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 18 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 19 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 20 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 21 - }, - { - "fields": { - "dest": 3, - "keep_output": false, - "pipelinestep": 1, - "source": 11, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 1 - }, - { - "fields": { - "dest": 4, - "keep_output": false, - "pipelinestep": 1, - "source": 12, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 2 - }, - { - "fields": { - "dest": 1, - "keep_output": false, - "pipelinestep": 2, - "source": 15, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 4 - }, - { - "fields": { - "dest": 11, - "keep_output": false, - "pipelinestep": 3, - "source": 13, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 5 - }, - { - "fields": { - "dest": 12, - "keep_output": false, - "pipelinestep": 3, - "source": 14, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 6 - }, - { - "fields": { - "dest": 7, - "keep_output": false, - "pipelinestep": 4, - "source": 2, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 7 - }, - { - "fields": { - "dest": 6, - "keep_output": false, - "pipelinestep": 4, - "source": 16, - "source_step": 2 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 8 - }, - { - "fields": { - "dest": 20, - "keep_output": false, - "pipelinestep": 5, - "source": 26, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 12 - }, - { - "fields": { - "dest": 20, - "keep_output": false, - "pipelinestep": 6, - "source": 21, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 13 - }, - { - "fields": { - "dest": 26, - "keep_output": false, - "pipelinestep": 7, - "source": 29, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 15 - }, - { - "fields": { - "dest": 26, - "keep_output": false, - "pipelinestep": 8, - "source": 28, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 16 - }, - { - "fields": { - "dest": 29, - "keep_output": false, - "pipelinestep": 9, - "source": 32, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 18 - }, - { - "fields": { - "dest": 29, - "keep_output": false, - "pipelinestep": 10, - "source": 31, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 19 - }, - { - "fields": { - "dest": 29, - "keep_output": false, - "pipelinestep": 11, - "source": 31, - "source_step": 2 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 20 - }, - { - "fields": { - "cable": 5, - "dest_pin": 21, - "source_pin": 23 - }, - "model": "pipeline.customcablewire", - "pk": 1 - }, - { - "fields": { - "cable": 5, - "dest_pin": 20, - "source_pin": 25 - }, - "model": "pipeline.customcablewire", - "pk": 2 - }, - { - "fields": { - "cable": 7, - "dest_pin": 21, - "source_pin": 20 - }, - "model": "pipeline.customcablewire", - "pk": 3 - }, - { - "fields": { - "cable": 7, - "dest_pin": 20, - "source_pin": 21 - }, - "model": "pipeline.customcablewire", - "pk": 4 - }, - { - "fields": { - "cable": 9, - "dest_pin": 21, - "source_pin": 24 - }, - "model": "pipeline.customcablewire", - "pk": 5 - }, - { - "fields": { - "cable": 9, - "dest_pin": 20, - "source_pin": 25 - }, - "model": "pipeline.customcablewire", - "pk": 6 - }, - { - "fields": { - "output_cdt": 14, - "output_idx": 1, - "output_name": "D1_out", - "pipeline": 4, - "source": 5, - "source_step": 1 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 3 - }, - { - "fields": { - "output_cdt": 12, - "output_idx": 1, - "output_name": "E1_out", - "pipeline": 5, - "source": 16, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 9 - }, - { - "fields": { - "output_cdt": 13, - "output_idx": 2, - "output_name": "E2_out", - "pipeline": 5, - "source": 8, - "source_step": 3 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 10 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 3, - "output_name": "E3_rawout", - "pipeline": 5, - "source": 10, - "source_step": 3 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 11 - }, - { - "fields": { - "output_cdt": 19, - "output_idx": 1, - "output_name": "basic_out", - "pipeline": 9, - "source": 21, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 14 - }, - { - "fields": { - "output_cdt": 19, - "output_idx": 1, - "output_name": "sub_out", - "pipeline": 10, - "source": 28, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 17 - }, - { - "fields": { - "output_cdt": 19, - "output_idx": 1, - "output_name": "top_out", - "pipeline": 11, - "source": 31, - "source_step": 3 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 21 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 1 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 2 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 3 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 4 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 5 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 6 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 7 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 8 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 9 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 10 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 11 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 12 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 13 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 14 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 15 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 16 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 17 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 18 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 19 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 20 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 21 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 22 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 23 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 24 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 25 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 26 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 28 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 29 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 31 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 32 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 34 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 2 - }, - "model": "transformation.xputstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 3 - }, - "model": "transformation.xputstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 4 - }, - "model": "transformation.xputstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": 5, - "min_row": null, - "transf_xput": 5 - }, - "model": "transformation.xputstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": null, - "min_row": null, - "transf_xput": 6 - }, - "model": "transformation.xputstructure", - "pk": 5 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 7 - }, - "model": "transformation.xputstructure", - "pk": 6 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 8 - }, - "model": "transformation.xputstructure", - "pk": 7 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 11 - }, - "model": "transformation.xputstructure", - "pk": 8 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 12 - }, - "model": "transformation.xputstructure", - "pk": 9 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": null, - "min_row": null, - "transf_xput": 13 - }, - "model": "transformation.xputstructure", - "pk": 10 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": 10, - "transf_xput": 14 - }, - "model": "transformation.xputstructure", - "pk": 11 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": 5, - "min_row": null, - "transf_xput": 16 - }, - "model": "transformation.xputstructure", - "pk": 12 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": 5, - "min_row": null, - "transf_xput": 17 - }, - "model": "transformation.xputstructure", - "pk": 13 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 18 - }, - "model": "transformation.xputstructure", - "pk": 14 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 20 - }, - "model": "transformation.xputstructure", - "pk": 15 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 21 - }, - "model": "transformation.xputstructure", - "pk": 16 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 22 - }, - "model": "transformation.xputstructure", - "pk": 17 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 23 - }, - "model": "transformation.xputstructure", - "pk": 18 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 26 - }, - "model": "transformation.xputstructure", - "pk": 19 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 28 - }, - "model": "transformation.xputstructure", - "pk": 21 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 29 - }, - "model": "transformation.xputstructure", - "pk": 22 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 31 - }, - "model": "transformation.xputstructure", - "pk": 24 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 32 - }, - "model": "transformation.xputstructure", - "pk": 25 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 34 - }, - "model": "transformation.xputstructure", - "pk": 27 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "A1_rawin", - "transformation": 1 - }, - "model": "transformation.transformationinput", - "pk": 1 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "B1_in", - "transformation": 2 - }, - "model": "transformation.transformationinput", - "pk": 3 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "B2_in", - "transformation": 2 - }, - "model": "transformation.transformationinput", - "pk": 4 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "C1_in", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 6 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "C2_in", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 7 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "D1_in", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 11 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "D2_in", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 12 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "E1_in", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 13 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "E2_in", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 14 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "E3_rawin", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 15 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "strings", - "transformation": 6 - }, - "model": "transformation.transformationinput", - "pk": 20 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "strings", - "transformation": 7 - }, - "model": "transformation.transformationinput", - "pk": 22 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "raw", - "transformation": 8 - }, - "model": "transformation.transformationinput", - "pk": 24 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "basic_in", - "transformation": 9 - }, - "model": "transformation.transformationinput", - "pk": 26 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "sub_in", - "transformation": 10 - }, - "model": "transformation.transformationinput", - "pk": 29 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "top_in", - "transformation": 11 - }, - "model": "transformation.transformationinput", - "pk": 32 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "A1_out", - "transformation": 1 - }, - "model": "transformation.transformationoutput", - "pk": 2 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "B1_out", - "transformation": 2 - }, - "model": "transformation.transformationoutput", - "pk": 5 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "C1_out", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 8 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "C2_rawout", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 9 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "C3_rawout", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 10 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "D1_out", - "transformation": 4 - }, - "model": "transformation.transformationoutput", - "pk": 16 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "E1_out", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 17 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "E2_out", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 18 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "E3_rawout", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 19 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "same_strings", - "transformation": 6 - }, - "model": "transformation.transformationoutput", - "pk": 21 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "untouched_strings", - "transformation": 7 - }, - "model": "transformation.transformationoutput", - "pk": 23 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "same_raw", - "transformation": 8 - }, - "model": "transformation.transformationoutput", - "pk": 25 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "basic_out", - "transformation": 9 - }, - "model": "transformation.transformationoutput", - "pk": 28 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "sub_out", - "transformation": 10 - }, - "model": "transformation.transformationoutput", - "pk": 31 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "top_out", - "transformation": 11 - }, - "model": "transformation.transformationoutput", - "pk": 34 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:00Z", - "email": "lennon@thebeatles.com", - "first_name": "", - "groups": [ - 1 - ], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$VWUS0gQFbjGx$qps1KQeGP14oMfGUIIHYxiip8P95QP35GO0B5bHooKs=", - "user_permissions": [], - "username": "john" - }, - "model": "auth.user", - "pk": 2 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:00Z", - "email": "starr@thebeatles.com", - "first_name": "", - "groups": [], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$7xPTl3d6fbRP$7Jkz/o7I6GSQCoPoKwYLPGF3XVP1rbPvta3nxJ2i91Q=", - "user_permissions": [], - "username": "ringo" - }, - "model": "auth.user", - "pk": 3 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:02Z", - "email": "bob@talabs.com", - "first_name": "", - "groups": [ - 1 - ], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$ehWkoaHUUF7H$dFKYkKNanz2koATDJmu1GriR5QD8XOd9CGTTChe+NQw=", - "user_permissions": [], - "username": "bob" - }, - "model": "auth.user", - "pk": 4 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:00Z", - "description": "String consisting of ACGTacgt", - "groups_allowed": [ - 1 - ], - "name": "DNANucSeq", - "prototype": null, - "restricts": [ - 1 - ], - "user": 2, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 8 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:00Z", - "description": "String consisting of ACGUacgu", - "groups_allowed": [ - 1 - ], - "name": "RNANucSeq", - "prototype": null, - "restricts": [ - 1 - ], - "user": 2, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 9 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:02Z", - "description": "sequences of ASCII characters", - "groups_allowed": [ - 1 - ], - "name": "my_string", - "prototype": null, - "restricts": [ - 1 - ], - "user": 4, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 10 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [], - "name": "", - "user": 2, - "users_allowed": [ - 3 - ] - }, - "model": "metadata.compounddatatype", - "pk": 6 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 7 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 8 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 9 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 10 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 11 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 12 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 13 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 14 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 15 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 16 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 17 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 18 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 4, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 19 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:48Z", - "groups_allowed": [ - 1 - ], - "name": "", - "parent_runstep": null, - "paused_by": null, - "pipeline": 11, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "/home/rliang/Documents/Kive/media/Sandboxes/userbob_run1_NWeZzc", - "start_time": "2000-01-01T00:00:04Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:52:14.720Z", - "user": 4, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 1 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:43Z", - "groups_allowed": [ - 1 - ], - "name": "", - "parent_runstep": 1, - "paused_by": null, - "pipeline": 10, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "", - "start_time": "2000-01-01T00:00:07Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:52:15.319Z", - "user": 4, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 2 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:41Z", - "groups_allowed": [ - 1 - ], - "name": "", - "parent_runstep": 2, - "paused_by": null, - "pipeline": 9, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "", - "start_time": "2000-01-01T00:00:07Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:52:15.363Z", - "user": 4, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 3 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:42Z", - "groups_allowed": [ - 1 - ], - "name": "", - "parent_runstep": 5, - "paused_by": null, - "pipeline": 9, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "", - "start_time": "2000-01-01T00:00:42Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:52:15.501Z", - "user": 4, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 4 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:45Z", - "groups_allowed": [ - 1 - ], - "name": "", - "parent_runstep": 8, - "paused_by": null, - "pipeline": 10, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "", - "start_time": "2000-01-01T00:00:43Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:52:15.627Z", - "user": 4, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 5 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:44Z", - "groups_allowed": [ - 1 - ], - "name": "", - "parent_runstep": 9, - "paused_by": null, - "pipeline": 9, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "", - "start_time": "2000-01-01T00:00:43Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:52:16.173Z", - "user": 4, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 6 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:45Z", - "groups_allowed": [ - 1 - ], - "name": "", - "parent_runstep": 12, - "paused_by": null, - "pipeline": 9, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "", - "start_time": "2000-01-01T00:00:44Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:52:16.266Z", - "user": 4, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 7 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:47Z", - "groups_allowed": [ - 1 - ], - "name": "", - "parent_runstep": 15, - "paused_by": null, - "pipeline": 10, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "", - "start_time": "2000-01-01T00:00:45Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:52:16.367Z", - "user": 4, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 8 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:46Z", - "groups_allowed": [ - 1 - ], - "name": "", - "parent_runstep": 16, - "paused_by": null, - "pipeline": 9, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "", - "start_time": "2000-01-01T00:00:45Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:52:16.405Z", - "user": 4, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 9 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:47Z", - "groups_allowed": [ - 1 - ], - "name": "", - "parent_runstep": 19, - "paused_by": null, - "pipeline": 9, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "", - "start_time": "2000-01-01T00:00:46Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:52:16.489Z", - "user": 4, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 10 - }, - { - "fields": { - "MD5_checksum": "ab7afe2f453d27fd168641a1f9271f26", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_triplet.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:11.635Z", - "name": "triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "02d5b66858d003b81ae3dc712b74e85c", - "_redacted": false, - "dataset_file": "Datasets/2017_04/doublet_cdt.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:11.714Z", - "name": "doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "29feb269edcac613d6eea432c14640bd", - "_redacted": false, - "dataset_file": "Datasets/2017_04/singlet_cdt_large.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:11.784Z", - "name": "singlet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 3 - }, - { - "fields": { - "MD5_checksum": "00fd50bfe7ac63e2165a442f4101e05c", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_singlet.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:11.835Z", - "name": "singlet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 4 - }, - { - "fields": { - "MD5_checksum": "7dc85e11b5c02e434af5bd3b3da9938e", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_raw.fasta", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:11.877Z", - "name": "raw_DS", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 5 - }, - { - "fields": { - "MD5_checksum": "542676b23e121d16db8d41ccdae65fd1", - "_redacted": false, - "dataset_file": "", - "date_created": "2000-01-01T00:00:01Z", - "description": "", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:11.914Z", - "name": "D1_in_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 6 - }, - { - "fields": { - "MD5_checksum": "56f9e5585b9d699829e2fda931d944f8", - "_redacted": false, - "dataset_file": "Datasets/2017_04/C1_in_triplet.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "triplet 3 rows", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:11.965Z", - "name": "C1_in_triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 7 - }, - { - "fields": { - "MD5_checksum": "10a4bd15d1300fcaf5a3bdad54ef1d28", - "_redacted": false, - "dataset_file": "", - "date_created": "2000-01-01T00:00:01Z", - "description": "", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:12.032Z", - "name": "C2_in_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 8 - }, - { - "fields": { - "MD5_checksum": "10a4bd15d1300fcaf5a3bdad54ef1d28", - "_redacted": false, - "dataset_file": "Datasets/2017_04/E11_32_output.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "result of E11_32 fed by doublet_cdt.csv", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:12.082Z", - "name": "E11_32 output doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 9 - }, - { - "fields": { - "MD5_checksum": "00fd50bfe7ac63e2165a442f4101e05c", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_singlet_oxkNUWB.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:12.141Z", - "name": "raw", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 10 - }, - { - "fields": { - "MD5_checksum": "7dc85e11b5c02e434af5bd3b3da9938e", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_raw_XRmFVVK.fasta", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:12.183Z", - "name": "C2_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 11 - }, - { - "fields": { - "MD5_checksum": "7dc85e11b5c02e434af5bd3b3da9938e", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_raw_CCI5cjK.fasta", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:12.212Z", - "name": "C3_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 12 - }, - { - "fields": { - "MD5_checksum": "56f9e5585b9d699829e2fda931d944f8", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_triplet_3_rows.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:12.247Z", - "name": "triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 13 - }, - { - "fields": { - "MD5_checksum": "83957f8aea1c75da9f7fbf9bc90da979", - "_redacted": false, - "dataset_file": "Datasets/2017_04/doublet_remuxed_from_t3r.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "doublet remuxed from triplet", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:12.314Z", - "name": "E1_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 14 - }, - { - "fields": { - "MD5_checksum": "d83866e4c23cd9503aa2f9fc565502b5", - "_redacted": false, - "dataset_file": "Datasets/2017_04/DNA_triplet.csv", - "date_created": "2000-01-01T00:00:02Z", - "description": "DNA triplet data", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:12.373Z", - "name": "DNA_triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 15 - }, - { - "fields": { - "MD5_checksum": "9eab0dfed5fcc89bd581f61995816ef0", - "_redacted": false, - "dataset_file": "Datasets/2017_04/E01_21_DNA_doublet.csv", - "date_created": "2000-01-01T00:00:02Z", - "description": "DNA doublet data coming from DNA_triplet.csv but remultiplexed according to cable E01_21", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:12.502Z", - "name": "E01_21_DNA_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 16 - }, - { - "fields": { - "MD5_checksum": "9b55b9431da6dba74ea61edf8e2ac044", - "_redacted": false, - "dataset_file": "Datasets/2017_04/E21_41_DNA_doublet.csv", - "date_created": "2000-01-01T00:00:02Z", - "description": "DNA doublet data coming from DNA_triplet.csv but remultiplexed according to cable E21_41", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:12.604Z", - "name": "E21_41_DNA_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 17 - }, - { - "fields": { - "MD5_checksum": "092ef90fe0201f4b002cd200f7754d45", - "_redacted": false, - "dataset_file": "Datasets/2017_04/tmpFsAQWW", - "date_created": "2000-01-01T00:00:04Z", - "description": "blahblahblah", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:14.657Z", - "name": "blahblah", - "user": 4, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 18 - }, - { - "fields": { - "MD5_checksum": "092ef90fe0201f4b002cd200f7754d45", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step1_same_strings.csv", - "date_created": "2000-01-01T00:00:19Z", - "description": "Generated data from a run of pipeline \"p_basic:1 (v1)\" started at 2017-04-25 00:52:18.053521+00:00 by bob\nrun: 3\nuser: bob\nstep: 1\noutput: same_strings", - "external_path": "", - "externalfiledirectory": null, - "file_source": 3, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:30.223Z", - "name": "run3_step1_outputsame_strings", - "user": 4, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 19 - }, - { - "fields": { - "MD5_checksum": "092ef90fe0201f4b002cd200f7754d45", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step2_same_strings.csv", - "date_created": "2000-01-01T00:00:34Z", - "description": "Generated data from a run of pipeline \"p_basic:1 (v1)\" started at 2017-04-25 00:52:18.053521+00:00 by bob\nrun: 3\nuser: bob\nstep: 2\noutput: same_strings", - "external_path": "", - "externalfiledirectory": null, - "file_source": 4, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:52:45.152Z", - "name": "run3_step2_outputsame_strings", - "user": 4, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 20 - }, - { - "fields": { - "description": "Just a CR", - "filename": "generic_script.py", - "groups_allowed": [ - 1 - ], - "name": "genericCR", - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 1 - }, - { - "fields": { - "description": "a script to do nothing", - "filename": "noop.sh", - "groups_allowed": [ - 1 - ], - "name": "noop", - "user": 4, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "e353a62fe8cb6a56d31b8c65d70b217d", - "coderesource": 1, - "content_file": "CodeResources/generic_script.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "desc", - "revision_name": "v1", - "revision_number": 1, - "revision_parent": null, - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "d218c00725d473639f347d456de381d8", - "coderesource": 2, - "content_file": "CodeResources/fdopen", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "1", - "revision_number": 1, - "revision_parent": null, - "user": 4, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 2 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 1 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 2, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 2 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 3, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 3 - }, - { - "fields": { - "driver": 2, - "family": 2, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 6 - }, - { - "fields": { - "driver": 2, - "family": 3, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 7 - }, - { - "fields": { - "driver": 2, - "family": 4, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 8 - }, - { - "fields": { - "description": "Holds methods A/B/C", - "groups_allowed": [ - 1 - ], - "name": "method_family", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 1 - }, - { - "fields": { - "description": "a method to do nothing to strings", - "groups_allowed": [ - 1 - ], - "name": "string noop", - "user": 4, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 2 - }, - { - "fields": { - "description": "a TOTALLY DIFFERENT method that TOTALLY does SOMETHING to strings by leaving them alone", - "groups_allowed": [ - 1 - ], - "name": "string trivial", - "user": 4, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 3 - }, - { - "fields": { - "description": "do nothing to raw data", - "groups_allowed": [ - 1 - ], - "name": "raw noop", - "user": 4, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 4 - }, - { - "fields": { - "description": "PF desc", - "groups_allowed": [ - 1 - ], - "name": "Pipeline_family", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 1 - }, - { - "fields": { - "description": "innermost pipeline", - "groups_allowed": [ - 1 - ], - "name": "p_basic", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 2 - }, - { - "fields": { - "description": "second-level pipeline", - "groups_allowed": [ - 1 - ], - "name": "p_sub", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 3 - }, - { - "fields": { - "description": "top-level pipeline", - "groups_allowed": [ - 1 - ], - "name": "p_top", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 4 - }, - { - "fields": { - "family": 1, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 4 - }, - { - "fields": { - "family": 1, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 2, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 5 - }, - { - "fields": { - "family": 2, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 9 - }, - { - "fields": { - "family": 3, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 10 - }, - { - "fields": { - "family": 4, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 11 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "A_desc", - "revision_name": "mA_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 1 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "B_desc", - "revision_name": "mB_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 2 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "C_desc", - "revision_name": "mC_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 3 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "D", - "revision_name": "pD_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 4 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "E", - "revision_name": "pE_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 4, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 6 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 4, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 7 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 4, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 8 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 9 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:03Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 10 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:03Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 11 - }, - { - "fields": { - "dataset": 1, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 3, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 3 - }, - { - "fields": { - "dataset": 4, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 4 - }, - { - "fields": { - "dataset": 6, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 5 - }, - { - "fields": { - "dataset": 7, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 6 - }, - { - "fields": { - "dataset": 8, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 7 - }, - { - "fields": { - "dataset": 9, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 8 - }, - { - "fields": { - "dataset": 10, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 9 - }, - { - "fields": { - "dataset": 13, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 10 - }, - { - "fields": { - "dataset": 14, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 11 - }, - { - "fields": { - "dataset": 15, - "end_time": "2000-01-01T00:00:02Z", - "execlog": null, - "start_time": "2000-01-01T00:00:02Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 12 - }, - { - "fields": { - "dataset": 16, - "end_time": "2000-01-01T00:00:02Z", - "execlog": null, - "start_time": "2000-01-01T00:00:02Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 13 - }, - { - "fields": { - "dataset": 17, - "end_time": "2000-01-01T00:00:02Z", - "execlog": null, - "start_time": "2000-01-01T00:00:02Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 14 - }, - { - "fields": { - "dataset": 18, - "end_time": "2000-01-01T00:00:04Z", - "execlog": null, - "start_time": "2000-01-01T00:00:04Z", - "user": 4 - }, - "model": "datachecking.contentchecklog", - "pk": 15 - }, - { - "fields": { - "dataset": 19, - "end_time": "2000-01-01T00:00:20Z", - "execlog": 3, - "start_time": "2000-01-01T00:00:20Z", - "user": 4 - }, - "model": "datachecking.contentchecklog", - "pk": 16 - }, - { - "fields": { - "dataset": 20, - "end_time": "2000-01-01T00:00:35Z", - "execlog": 5, - "start_time": "2000-01-01T00:00:35Z", - "user": 4 - }, - "model": "datachecking.contentchecklog", - "pk": 17 - }, - { - "fields": { - "dataset": 18, - "end_time": "2000-01-01T00:00:06Z", - "execlog": null, - "read_failed": false, - "runcomponent": 22, - "start_time": "2000-01-01T00:00:06Z", - "user": 4 - }, - "model": "datachecking.integritychecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 19, - "end_time": "2000-01-01T00:00:20Z", - "execlog": 3, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:20Z", - "user": 4 - }, - "model": "datachecking.integritychecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 20, - "end_time": "2000-01-01T00:00:35Z", - "execlog": 5, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:35Z", - "user": 4 - }, - "model": "datachecking.integritychecklog", - "pk": 3 - } -] \ No newline at end of file diff --git a/kive/portal/fixtures/demo.json b/kive/portal/fixtures/demo.json deleted file mode 100644 index 013549678..000000000 --- a/kive/portal/fixtures/demo.json +++ /dev/null @@ -1,3471 +0,0 @@ -[ - { - "fields": { - "dataset": 1, - "index": 1, - "run": 1 - }, - "model": "archive.runinput", - "pk": 1 - }, - { - "fields": { - "dataset": 3, - "index": 1, - "run": 2 - }, - "model": "archive.runinput", - "pk": 2 - }, - { - "fields": { - "dataset": 4, - "index": 2, - "run": 2 - }, - "model": "archive.runinput", - "pk": 3 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:14Z", - "execrecord": 2, - "reused": false, - "start_time": "2000-01-01T00:00:00Z" - }, - "model": "archive.runcomponent", - "pk": 1 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:00Z", - "execrecord": 1, - "reused": false, - "start_time": "2000-01-01T00:00:00Z" - }, - "model": "archive.runcomponent", - "pk": 2 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:16Z", - "execrecord": 3, - "reused": false, - "start_time": "2000-01-01T00:00:16Z" - }, - "model": "archive.runcomponent", - "pk": 3 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 5, - "end_time": "2000-01-01T00:00:35Z", - "execrecord": 6, - "reused": false, - "start_time": "2000-01-01T00:00:21Z" - }, - "model": "archive.runcomponent", - "pk": 4 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 4, - "end_time": null, - "execrecord": null, - "reused": null, - "start_time": null - }, - "model": "archive.runcomponent", - "pk": 5 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 4, - "end_time": null, - "execrecord": null, - "reused": null, - "start_time": null - }, - "model": "archive.runcomponent", - "pk": 6 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 4, - "end_time": null, - "execrecord": null, - "reused": null, - "start_time": null - }, - "model": "archive.runcomponent", - "pk": 7 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 4, - "end_time": null, - "execrecord": null, - "reused": null, - "start_time": null - }, - "model": "archive.runcomponent", - "pk": 8 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 4, - "end_time": null, - "execrecord": null, - "reused": null, - "start_time": null - }, - "model": "archive.runcomponent", - "pk": 9 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:21Z", - "execrecord": 4, - "reused": false, - "start_time": "2000-01-01T00:00:21Z" - }, - "model": "archive.runcomponent", - "pk": 10 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:22Z", - "execrecord": 5, - "reused": false, - "start_time": "2000-01-01T00:00:21Z" - }, - "model": "archive.runcomponent", - "pk": 11 - }, - { - "fields": { - "pipelinestep": 1, - "run": 1 - }, - "model": "archive.runstep", - "pk": 1 - }, - { - "fields": { - "pipelinestep": 2, - "run": 2 - }, - "model": "archive.runstep", - "pk": 4 - }, - { - "fields": { - "pipelinestep": 3, - "run": 2 - }, - "model": "archive.runstep", - "pk": 5 - }, - { - "fields": { - "pipelinestep": 4, - "run": 2 - }, - "model": "archive.runstep", - "pk": 6 - }, - { - "fields": { - "pipelinestep": 5, - "run": 2 - }, - "model": "archive.runstep", - "pk": 7 - }, - { - "fields": { - "pipelinestep": 6, - "run": 2 - }, - "model": "archive.runstep", - "pk": 8 - }, - { - "fields": { - "pipelinestep": 7, - "run": 2 - }, - "model": "archive.runstep", - "pk": 9 - }, - { - "fields": { - "PSIC": 1, - "dest_runstep": 1 - }, - "model": "archive.runsic", - "pk": 2 - }, - { - "fields": { - "PSIC": 3, - "dest_runstep": 4 - }, - "model": "archive.runsic", - "pk": 10 - }, - { - "fields": { - "PSIC": 4, - "dest_runstep": 4 - }, - "model": "archive.runsic", - "pk": 11 - }, - { - "fields": { - "pipelineoutputcable": 2, - "run": 1 - }, - "model": "archive.runoutputcable", - "pk": 3 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:00Z", - "invoking_record": 2, - "record": 2, - "start_time": "2000-01-01T00:00:00Z" - }, - "model": "archive.execlog", - "pk": 1 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:01Z", - "invoking_record": 1, - "record": 1, - "start_time": "2000-01-01T00:00:01Z" - }, - "model": "archive.execlog", - "pk": 2 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:16Z", - "invoking_record": 3, - "record": 3, - "start_time": "2000-01-01T00:00:16Z" - }, - "model": "archive.execlog", - "pk": 3 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:21Z", - "invoking_record": 10, - "record": 10, - "start_time": "2000-01-01T00:00:21Z" - }, - "model": "archive.execlog", - "pk": 4 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:22Z", - "invoking_record": 11, - "record": 11, - "start_time": "2000-01-01T00:00:22Z" - }, - "model": "archive.execlog", - "pk": 5 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:22Z", - "invoking_record": 4, - "record": 4, - "start_time": "2000-01-01T00:00:22Z" - }, - "model": "archive.execlog", - "pk": 6 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step1_stderr_slurmIDJ_nodeN.txt", - "error_redacted": false, - "execlog": 2, - "install_failed": false, - "output_log": "Logs/step1_stdout_slurmIDJ_nodeN.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 1 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step1_stderr_slurmIDJ_nodeN_MAzNZpx.txt", - "error_redacted": false, - "execlog": 6, - "install_failed": false, - "output_log": "Logs/step1_stdout_slurmIDJ_nodeN_NRGCddL.txt", - "output_redacted": false, - "return_code": 1 - }, - "model": "archive.methodoutput", - "pk": 2 - }, - { - "fields": { - "generator": 1 - }, - "model": "librarian.execrecord", - "pk": 1 - }, - { - "fields": { - "generator": 2 - }, - "model": "librarian.execrecord", - "pk": 2 - }, - { - "fields": { - "generator": 3 - }, - "model": "librarian.execrecord", - "pk": 3 - }, - { - "fields": { - "generator": 4 - }, - "model": "librarian.execrecord", - "pk": 4 - }, - { - "fields": { - "generator": 5 - }, - "model": "librarian.execrecord", - "pk": 5 - }, - { - "fields": { - "generator": 6 - }, - "model": "librarian.execrecord", - "pk": 6 - }, - { - "fields": { - "dataset": 1, - "execrecord": 1, - "generic_input": 3 - }, - "model": "librarian.execrecordin", - "pk": 1 - }, - { - "fields": { - "dataset": 1, - "execrecord": 2, - "generic_input": 1 - }, - "model": "librarian.execrecordin", - "pk": 2 - }, - { - "fields": { - "dataset": 2, - "execrecord": 3, - "generic_input": 2 - }, - "model": "librarian.execrecordin", - "pk": 3 - }, - { - "fields": { - "dataset": 3, - "execrecord": 4, - "generic_input": 34 - }, - "model": "librarian.execrecordin", - "pk": 4 - }, - { - "fields": { - "dataset": 4, - "execrecord": 5, - "generic_input": 35 - }, - "model": "librarian.execrecordin", - "pk": 5 - }, - { - "fields": { - "dataset": 3, - "execrecord": 6, - "generic_input": 5 - }, - "model": "librarian.execrecordin", - "pk": 6 - }, - { - "fields": { - "dataset": 4, - "execrecord": 6, - "generic_input": 6 - }, - "model": "librarian.execrecordin", - "pk": 7 - }, - { - "fields": { - "dataset": 1, - "execrecord": 1, - "generic_output": 1 - }, - "model": "librarian.execrecordout", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "execrecord": 2, - "generic_output": 2 - }, - "model": "librarian.execrecordout", - "pk": 2 - }, - { - "fields": { - "dataset": 2, - "execrecord": 3, - "generic_output": 4 - }, - "model": "librarian.execrecordout", - "pk": 3 - }, - { - "fields": { - "dataset": 3, - "execrecord": 4, - "generic_output": 5 - }, - "model": "librarian.execrecordout", - "pk": 4 - }, - { - "fields": { - "dataset": 4, - "execrecord": 5, - "generic_output": 6 - }, - "model": "librarian.execrecordout", - "pk": 5 - }, - { - "fields": { - "dataset": 5, - "execrecord": 6, - "generic_output": 7 - }, - "model": "librarian.execrecordout", - "pk": 6 - }, - { - "fields": { - "filename": "projects.json", - "method": 3, - "path": "micall/core", - "requirement": 12 - }, - "model": "method.methoddependency", - "pk": 1 - }, - { - "fields": { - "filename": "externals.py", - "method": 3, - "path": "micall/utils", - "requirement": 10 - }, - "model": "method.methoddependency", - "pk": 2 - }, - { - "fields": { - "filename": "miseq_logging.py", - "method": 3, - "path": "micall/core", - "requirement": 7 - }, - "model": "method.methoddependency", - "pk": 3 - }, - { - "fields": { - "filename": "project_scoring.json", - "method": 3, - "path": "micall/core", - "requirement": 16 - }, - "model": "method.methoddependency", - "pk": 4 - }, - { - "fields": { - "filename": "project_config.py", - "method": 3, - "path": "micall/core", - "requirement": 15 - }, - "model": "method.methoddependency", - "pk": 5 - }, - { - "fields": { - "filename": "__init__.py", - "method": 3, - "path": "micall/utils", - "requirement": 3 - }, - "model": "method.methoddependency", - "pk": 6 - }, - { - "fields": { - "filename": "__init__.py", - "method": 3, - "path": "micall/core", - "requirement": 3 - }, - "model": "method.methoddependency", - "pk": 7 - }, - { - "fields": { - "filename": "__init__.py", - "method": 3, - "path": "micall", - "requirement": 3 - }, - "model": "method.methoddependency", - "pk": 8 - }, - { - "fields": { - "filename": "prelim_map.py", - "method": 4, - "path": "micall/core", - "requirement": 2 - }, - "model": "method.methoddependency", - "pk": 9 - }, - { - "fields": { - "filename": "translation.py", - "method": 4, - "path": "micall/utils", - "requirement": 8 - }, - "model": "method.methoddependency", - "pk": 10 - }, - { - "fields": { - "filename": "projects.json", - "method": 4, - "path": "micall/core", - "requirement": 12 - }, - "model": "method.methoddependency", - "pk": 11 - }, - { - "fields": { - "filename": "externals.py", - "method": 4, - "path": "micall/utils", - "requirement": 10 - }, - "model": "method.methoddependency", - "pk": 12 - }, - { - "fields": { - "filename": "project_config.py", - "method": 4, - "path": "micall/core", - "requirement": 15 - }, - "model": "method.methoddependency", - "pk": 13 - }, - { - "fields": { - "filename": "sam2aln.py", - "method": 4, - "path": "micall/core", - "requirement": 5 - }, - "model": "method.methoddependency", - "pk": 14 - }, - { - "fields": { - "filename": "miseq_logging.py", - "method": 4, - "path": "micall/core", - "requirement": 7 - }, - "model": "method.methoddependency", - "pk": 15 - }, - { - "fields": { - "filename": "project_scoring.json", - "method": 4, - "path": "micall/core", - "requirement": 16 - }, - "model": "method.methoddependency", - "pk": 16 - }, - { - "fields": { - "filename": "__init__.py", - "method": 4, - "path": "micall/utils", - "requirement": 3 - }, - "model": "method.methoddependency", - "pk": 17 - }, - { - "fields": { - "filename": "__init__.py", - "method": 4, - "path": "micall/core", - "requirement": 3 - }, - "model": "method.methoddependency", - "pk": 18 - }, - { - "fields": { - "filename": "__init__.py", - "method": 4, - "path": "micall", - "requirement": 3 - }, - "model": "method.methoddependency", - "pk": 19 - }, - { - "fields": { - "filename": "projects.json", - "method": 6, - "path": "micall/core", - "requirement": 12 - }, - "model": "method.methoddependency", - "pk": 20 - }, - { - "fields": { - "filename": "translation.py", - "method": 6, - "path": "micall/utils", - "requirement": 8 - }, - "model": "method.methoddependency", - "pk": 21 - }, - { - "fields": { - "filename": "miseq_logging.py", - "method": 6, - "path": "micall/core", - "requirement": 7 - }, - "model": "method.methoddependency", - "pk": 22 - }, - { - "fields": { - "filename": "project_scoring.json", - "method": 6, - "path": "micall/core", - "requirement": 16 - }, - "model": "method.methoddependency", - "pk": 23 - }, - { - "fields": { - "filename": "project_config.py", - "method": 6, - "path": "micall/core", - "requirement": 15 - }, - "model": "method.methoddependency", - "pk": 24 - }, - { - "fields": { - "filename": "__init__.py", - "method": 6, - "path": "micall/utils", - "requirement": 3 - }, - "model": "method.methoddependency", - "pk": 25 - }, - { - "fields": { - "filename": "__init__.py", - "method": 6, - "path": "micall/core", - "requirement": 3 - }, - "model": "method.methoddependency", - "pk": 26 - }, - { - "fields": { - "filename": "__init__.py", - "method": 6, - "path": "micall", - "requirement": 3 - }, - "model": "method.methoddependency", - "pk": 27 - }, - { - "fields": { - "filename": "sam2aln.py", - "method": 7, - "path": "micall/core", - "requirement": 5 - }, - "model": "method.methoddependency", - "pk": 28 - }, - { - "fields": { - "filename": "pssm_lib.py", - "method": 7, - "path": "micall/g2p", - "requirement": 4 - }, - "model": "method.methoddependency", - "pk": 29 - }, - { - "fields": { - "filename": "translation.py", - "method": 7, - "path": "micall/utils", - "requirement": 8 - }, - "model": "method.methoddependency", - "pk": 30 - }, - { - "fields": { - "filename": "g2p.matrix", - "method": 7, - "path": "micall/g2p", - "requirement": 14 - }, - "model": "method.methoddependency", - "pk": 31 - }, - { - "fields": { - "filename": "g2p_fpr.txt", - "method": 7, - "path": "micall/g2p", - "requirement": 17 - }, - "model": "method.methoddependency", - "pk": 32 - }, - { - "fields": { - "filename": "__init__.py", - "method": 7, - "path": "micall/g2p", - "requirement": 3 - }, - "model": "method.methoddependency", - "pk": 33 - }, - { - "fields": { - "filename": "__init__.py", - "method": 7, - "path": "micall/utils", - "requirement": 3 - }, - "model": "method.methoddependency", - "pk": 34 - }, - { - "fields": { - "filename": "__init__.py", - "method": 7, - "path": "micall/core", - "requirement": 3 - }, - "model": "method.methoddependency", - "pk": 35 - }, - { - "fields": { - "filename": "__init__.py", - "method": 7, - "path": "micall", - "requirement": 3 - }, - "model": "method.methoddependency", - "pk": 36 - }, - { - "fields": { - "filename": "translation.py", - "method": 8, - "path": "micall/utils", - "requirement": 8 - }, - "model": "method.methoddependency", - "pk": 37 - }, - { - "fields": { - "filename": "projects.json", - "method": 8, - "path": "micall/core", - "requirement": 12 - }, - "model": "method.methoddependency", - "pk": 38 - }, - { - "fields": { - "filename": "miseq_logging.py", - "method": 8, - "path": "micall/core", - "requirement": 7 - }, - "model": "method.methoddependency", - "pk": 39 - }, - { - "fields": { - "filename": "aln2counts.py", - "method": 8, - "path": "micall/core", - "requirement": 6 - }, - "model": "method.methoddependency", - "pk": 40 - }, - { - "fields": { - "filename": "project_config.py", - "method": 8, - "path": "micall/core", - "requirement": 15 - }, - "model": "method.methoddependency", - "pk": 41 - }, - { - "fields": { - "filename": "project_scoring.json", - "method": 8, - "path": "micall/core", - "requirement": 16 - }, - "model": "method.methoddependency", - "pk": 42 - }, - { - "fields": { - "filename": "__init__.py", - "method": 8, - "path": "micall/utils", - "requirement": 3 - }, - "model": "method.methoddependency", - "pk": 43 - }, - { - "fields": { - "filename": "__init__.py", - "method": 8, - "path": "micall/core", - "requirement": 3 - }, - "model": "method.methoddependency", - "pk": 44 - }, - { - "fields": { - "filename": "__init__.py", - "method": 8, - "path": "micall", - "requirement": 3 - }, - "model": "method.methoddependency", - "pk": 45 - }, - { - "fields": { - "fill_colour": "", - "name": "sums_and_products", - "outputs_to_delete": [], - "pipeline": 2, - "step_num": 1, - "transformation": 1, - "x": 0.5, - "y": 0.5 - }, - "model": "pipeline.pipelinestep", - "pk": 1 - }, - { - "fields": { - "fill_colour": "", - "name": "prelim", - "outputs_to_delete": [], - "pipeline": 9, - "step_num": 1, - "transformation": 3, - "x": 0.3, - "y": 0.3 - }, - "model": "pipeline.pipelinestep", - "pk": 2 - }, - { - "fields": { - "fill_colour": "", - "name": "remap", - "outputs_to_delete": [], - "pipeline": 9, - "step_num": 2, - "transformation": 4, - "x": 0.4, - "y": 0.4 - }, - "model": "pipeline.pipelinestep", - "pk": 3 - }, - { - "fields": { - "fill_colour": "", - "name": "sam2aln", - "outputs_to_delete": [], - "pipeline": 9, - "step_num": 3, - "transformation": 5, - "x": 0.5, - "y": 0.5 - }, - "model": "pipeline.pipelinestep", - "pk": 4 - }, - { - "fields": { - "fill_colour": "", - "name": "aln2counts", - "outputs_to_delete": [], - "pipeline": 9, - "step_num": 4, - "transformation": 6, - "x": 0.6, - "y": 0.6 - }, - "model": "pipeline.pipelinestep", - "pk": 5 - }, - { - "fields": { - "fill_colour": "", - "name": "sam_g2p", - "outputs_to_delete": [], - "pipeline": 9, - "step_num": 5, - "transformation": 7, - "x": 0.7, - "y": 0.7 - }, - "model": "pipeline.pipelinestep", - "pk": 6 - }, - { - "fields": { - "fill_colour": "", - "name": "coverage_plots", - "outputs_to_delete": [], - "pipeline": 9, - "step_num": 6, - "transformation": 8, - "x": 0.8, - "y": 0.8 - }, - "model": "pipeline.pipelinestep", - "pk": 7 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 1 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 2 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 3 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 4 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 5 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 6 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 7 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 8 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 9 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 10 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 11 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 12 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 13 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 14 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 15 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 16 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 17 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 18 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 19 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 20 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 21 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 22 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 23 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 24 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 25 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 26 - }, - { - "fields": { - "dest": 1, - "keep_output": false, - "pipelinestep": 1, - "source": 3, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 1 - }, - { - "fields": { - "dest": 5, - "keep_output": false, - "pipelinestep": 2, - "source": 34, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 3 - }, - { - "fields": { - "dest": 6, - "keep_output": false, - "pipelinestep": 2, - "source": 35, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 4 - }, - { - "fields": { - "dest": 8, - "keep_output": false, - "pipelinestep": 3, - "source": 34, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 5 - }, - { - "fields": { - "dest": 9, - "keep_output": false, - "pipelinestep": 3, - "source": 35, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 6 - }, - { - "fields": { - "dest": 10, - "keep_output": false, - "pipelinestep": 3, - "source": 7, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 7 - }, - { - "fields": { - "dest": 16, - "keep_output": false, - "pipelinestep": 4, - "source": 11, - "source_step": 2 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 8 - }, - { - "fields": { - "dest": 20, - "keep_output": false, - "pipelinestep": 5, - "source": 17, - "source_step": 3 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 9 - }, - { - "fields": { - "dest": 27, - "keep_output": false, - "pipelinestep": 6, - "source": 11, - "source_step": 2 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 10 - }, - { - "fields": { - "dest": 28, - "keep_output": false, - "pipelinestep": 6, - "source": 21, - "source_step": 4 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 11 - }, - { - "fields": { - "dest": 31, - "keep_output": false, - "pipelinestep": 7, - "source": 22, - "source_step": 4 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 12 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 1, - "output_name": "sums_and_products", - "pipeline": 2, - "source": 2, - "source_step": 1 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 2 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 1, - "output_name": "remap_counts", - "pipeline": 9, - "source": 12, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 13 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 2, - "output_name": "remap_conseq", - "pipeline": 9, - "source": 13, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 14 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 3, - "output_name": "unmappedfor", - "pipeline": 9, - "source": 14, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 15 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 4, - "output_name": "unmappedrev", - "pipeline": 9, - "source": 15, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 16 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 5, - "output_name": "conseq_insertions", - "pipeline": 9, - "source": 18, - "source_step": 3 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 17 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 6, - "output_name": "failed_read", - "pipeline": 9, - "source": 19, - "source_step": 3 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 18 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 7, - "output_name": "coord_ins", - "pipeline": 9, - "source": 23, - "source_step": 4 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 19 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 8, - "output_name": "conseq", - "pipeline": 9, - "source": 24, - "source_step": 4 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 20 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 9, - "output_name": "failed_align", - "pipeline": 9, - "source": 25, - "source_step": 4 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 21 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 10, - "output_name": "nuc_variants", - "pipeline": 9, - "source": 26, - "source_step": 4 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 22 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 11, - "output_name": "g2p", - "pipeline": 9, - "source": 29, - "source_step": 5 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 23 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 12, - "output_name": "g2p_summary", - "pipeline": 9, - "source": 30, - "source_step": 5 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 24 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 13, - "output_name": "coverage_scores_csv", - "pipeline": 9, - "source": 32, - "source_step": 6 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 25 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 14, - "output_name": "coverage_maps_tar", - "pipeline": 9, - "source": 33, - "source_step": 6 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 26 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 1 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 2 - }, - { - "fields": { - "x": 0.25, - "y": 0.25 - }, - "model": "transformation.transformationxput", - "pk": 3 - }, - { - "fields": { - "x": 0.75, - "y": 0.75 - }, - "model": "transformation.transformationxput", - "pk": 4 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 5 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 6 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 7 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 8 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 9 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 10 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 11 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 12 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 13 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 14 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 15 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 16 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 17 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 18 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 19 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 20 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 21 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 22 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 23 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 24 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 25 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 26 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 27 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 28 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 29 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 30 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 31 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 32 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 33 - }, - { - "fields": { - "x": 0.1, - "y": 0.1 - }, - "model": "transformation.transformationxput", - "pk": 34 - }, - { - "fields": { - "x": 0.2, - "y": 0.2 - }, - "model": "transformation.transformationxput", - "pk": 35 - }, - { - "fields": { - "x": 0.95, - "y": 0.9 - }, - "model": "transformation.transformationxput", - "pk": 36 - }, - { - "fields": { - "x": 0.9, - "y": 0.9 - }, - "model": "transformation.transformationxput", - "pk": 37 - }, - { - "fields": { - "x": 0.85, - "y": 0.9 - }, - "model": "transformation.transformationxput", - "pk": 38 - }, - { - "fields": { - "x": 0.8, - "y": 0.9 - }, - "model": "transformation.transformationxput", - "pk": 39 - }, - { - "fields": { - "x": 0.75, - "y": 0.9 - }, - "model": "transformation.transformationxput", - "pk": 40 - }, - { - "fields": { - "x": 0.7, - "y": 0.9 - }, - "model": "transformation.transformationxput", - "pk": 41 - }, - { - "fields": { - "x": 0.65, - "y": 0.9 - }, - "model": "transformation.transformationxput", - "pk": 42 - }, - { - "fields": { - "x": 0.6, - "y": 0.9 - }, - "model": "transformation.transformationxput", - "pk": 43 - }, - { - "fields": { - "x": 0.55, - "y": 0.9 - }, - "model": "transformation.transformationxput", - "pk": 44 - }, - { - "fields": { - "x": 0.5, - "y": 0.9 - }, - "model": "transformation.transformationxput", - "pk": 45 - }, - { - "fields": { - "x": 0.45, - "y": 0.9 - }, - "model": "transformation.transformationxput", - "pk": 46 - }, - { - "fields": { - "x": 0.4, - "y": 0.9 - }, - "model": "transformation.transformationxput", - "pk": 47 - }, - { - "fields": { - "x": 0.35, - "y": 0.9 - }, - "model": "transformation.transformationxput", - "pk": 48 - }, - { - "fields": { - "x": 0.3, - "y": 0.9 - }, - "model": "transformation.transformationxput", - "pk": 49 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "pairs", - "transformation": 1 - }, - "model": "transformation.transformationinput", - "pk": 1 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "pairs", - "transformation": 2 - }, - "model": "transformation.transformationinput", - "pk": 3 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "forward", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 5 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "reverse", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 6 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "forward", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 8 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "reverse", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 9 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "prelim_out", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 10 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "remap_csv", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 16 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "aligned", - "transformation": 6 - }, - "model": "transformation.transformationinput", - "pk": 20 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "remap_csv", - "transformation": 7 - }, - "model": "transformation.transformationinput", - "pk": 27 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "nuc", - "transformation": 7 - }, - "model": "transformation.transformationinput", - "pk": 28 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "amino", - "transformation": 8 - }, - "model": "transformation.transformationinput", - "pk": 31 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "forward", - "transformation": 9 - }, - "model": "transformation.transformationinput", - "pk": 34 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "reverse", - "transformation": 9 - }, - "model": "transformation.transformationinput", - "pk": 35 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "sums_and_products", - "transformation": 1 - }, - "model": "transformation.transformationoutput", - "pk": 2 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "sums_and_products", - "transformation": 2 - }, - "model": "transformation.transformationoutput", - "pk": 4 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "prelim_out", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 7 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "remap_csv", - "transformation": 4 - }, - "model": "transformation.transformationoutput", - "pk": 11 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "remap_counts", - "transformation": 4 - }, - "model": "transformation.transformationoutput", - "pk": 12 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "remap_conseq", - "transformation": 4 - }, - "model": "transformation.transformationoutput", - "pk": 13 - }, - { - "fields": { - "dataset_idx": 4, - "dataset_name": "unmappedfor", - "transformation": 4 - }, - "model": "transformation.transformationoutput", - "pk": 14 - }, - { - "fields": { - "dataset_idx": 5, - "dataset_name": "unmappedrev", - "transformation": 4 - }, - "model": "transformation.transformationoutput", - "pk": 15 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "aligned", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 17 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "conseq_insertions", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 18 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "failed_read", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 19 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "nuc", - "transformation": 6 - }, - "model": "transformation.transformationoutput", - "pk": 21 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "amino", - "transformation": 6 - }, - "model": "transformation.transformationoutput", - "pk": 22 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "coord_ins", - "transformation": 6 - }, - "model": "transformation.transformationoutput", - "pk": 23 - }, - { - "fields": { - "dataset_idx": 4, - "dataset_name": "conseq", - "transformation": 6 - }, - "model": "transformation.transformationoutput", - "pk": 24 - }, - { - "fields": { - "dataset_idx": 5, - "dataset_name": "failed_align", - "transformation": 6 - }, - "model": "transformation.transformationoutput", - "pk": 25 - }, - { - "fields": { - "dataset_idx": 6, - "dataset_name": "nuc_variants", - "transformation": 6 - }, - "model": "transformation.transformationoutput", - "pk": 26 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "g2p", - "transformation": 7 - }, - "model": "transformation.transformationoutput", - "pk": 29 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "g2p_summary", - "transformation": 7 - }, - "model": "transformation.transformationoutput", - "pk": 30 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "coverage_scores_csv", - "transformation": 8 - }, - "model": "transformation.transformationoutput", - "pk": 32 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "coverage_maps_tar", - "transformation": 8 - }, - "model": "transformation.transformationoutput", - "pk": 33 - }, - { - "fields": { - "dataset_idx": 14, - "dataset_name": "coverage_maps_tar", - "transformation": 9 - }, - "model": "transformation.transformationoutput", - "pk": 36 - }, - { - "fields": { - "dataset_idx": 13, - "dataset_name": "coverage_scores_csv", - "transformation": 9 - }, - "model": "transformation.transformationoutput", - "pk": 37 - }, - { - "fields": { - "dataset_idx": 12, - "dataset_name": "g2p_summary", - "transformation": 9 - }, - "model": "transformation.transformationoutput", - "pk": 38 - }, - { - "fields": { - "dataset_idx": 11, - "dataset_name": "g2p", - "transformation": 9 - }, - "model": "transformation.transformationoutput", - "pk": 39 - }, - { - "fields": { - "dataset_idx": 10, - "dataset_name": "nuc_variants", - "transformation": 9 - }, - "model": "transformation.transformationoutput", - "pk": 40 - }, - { - "fields": { - "dataset_idx": 9, - "dataset_name": "failed_align", - "transformation": 9 - }, - "model": "transformation.transformationoutput", - "pk": 41 - }, - { - "fields": { - "dataset_idx": 8, - "dataset_name": "conseq", - "transformation": 9 - }, - "model": "transformation.transformationoutput", - "pk": 42 - }, - { - "fields": { - "dataset_idx": 7, - "dataset_name": "coord_ins", - "transformation": 9 - }, - "model": "transformation.transformationoutput", - "pk": 43 - }, - { - "fields": { - "dataset_idx": 6, - "dataset_name": "failed_read", - "transformation": 9 - }, - "model": "transformation.transformationoutput", - "pk": 44 - }, - { - "fields": { - "dataset_idx": 5, - "dataset_name": "conseq_insertions", - "transformation": 9 - }, - "model": "transformation.transformationoutput", - "pk": 45 - }, - { - "fields": { - "dataset_idx": 4, - "dataset_name": "unmappedrev", - "transformation": 9 - }, - "model": "transformation.transformationoutput", - "pk": 46 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "unmappedfor", - "transformation": 9 - }, - "model": "transformation.transformationoutput", - "pk": 47 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "remap_conseq", - "transformation": 9 - }, - "model": "transformation.transformationoutput", - "pk": 48 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "remap_counts", - "transformation": 9 - }, - "model": "transformation.transformationoutput", - "pk": 49 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:17Z", - "groups_allowed": [], - "name": "", - "parent_runstep": null, - "paused_by": null, - "pipeline": 2, - "priority": 0, - "purged": true, - "runbatch": null, - "sandbox_path": "/home/rliang/Documents/Kive/media/Sandboxes/userkive_run1_08TJQE", - "start_time": "2000-01-01T00:00:00Z", - "stopped_by": null, - "time_queued": "2017-04-25T01:00:39.339Z", - "user": 1, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 1 - }, - { - "fields": { - "_runstate": 7, - "description": "", - "end_time": "2000-01-01T00:00:37Z", - "groups_allowed": [], - "name": "", - "parent_runstep": null, - "paused_by": null, - "pipeline": 9, - "priority": 0, - "purged": true, - "runbatch": null, - "sandbox_path": "/home/rliang/Documents/Kive/media/Sandboxes/userkive_run2_rBOMWE", - "start_time": "2000-01-01T00:00:20Z", - "stopped_by": null, - "time_queued": "2017-04-25T01:00:59.723Z", - "user": 1, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "e093c9eac5d0cae832d4e2071ccfdc5f", - "_redacted": false, - "dataset_file": "Datasets/2017_04/pairs.csv", - "date_created": "2000-01-01T00:00:00Z", - "description": "", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T01:00:39.325Z", - "name": "pairs", - "user": 1, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "f5e2c5378e2c92de9207d30227ddc888", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step1_sums_and_products.raw", - "date_created": "2000-01-01T00:00:14Z", - "description": "Generated data from a run of pipeline \"sums and products:1 (sums and products)\" started at 2017-04-25 01:00:39.449882+00:00 by kive\nrun: 1\nuser: kive\nstep: 1\noutput: sums_and_products", - "external_path": "", - "externalfiledirectory": null, - "file_source": 1, - "groups_allowed": [], - "last_time_checked": "2017-04-25T01:00:53.253Z", - "name": "run1_step1_outputsums_and_products", - "user": 1, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "5bf4c20640428a4578d1989425123514", - "_redacted": false, - "dataset_file": "Datasets/2017_04/1234A-V3LOOP_S1_L001_R1_001.fastq", - "date_created": "2000-01-01T00:00:20Z", - "description": "", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T01:00:59.703Z", - "name": "1234A-V3LOOP_S1_L001_R1_001.fastq", - "user": 1, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 3 - }, - { - "fields": { - "MD5_checksum": "104f5404f7f213b15d3ec7d5f72ee004", - "_redacted": false, - "dataset_file": "Datasets/2017_04/1234A-V3LOOP_S1_L001_R2_001.fastq", - "date_created": "2000-01-01T00:00:20Z", - "description": "", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T01:00:59.713Z", - "name": "1234A-V3LOOP_S1_L001_R2_001.fastq", - "user": 1, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 4 - }, - { - "fields": { - "MD5_checksum": "d41d8cd98f00b204e9800998ecf8427e", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step1_prelim_out.raw", - "date_created": "2000-01-01T00:00:34Z", - "description": "Generated data from a run of pipeline \"MiCallDemo:1 (MiCall Demo 2016-09)\" started at 2017-04-25 01:00:59.860887+00:00 by kive\nrun: 2\nuser: kive\nstep: 1\noutput: prelim_out", - "external_path": "", - "externalfiledirectory": null, - "file_source": 4, - "groups_allowed": [], - "last_time_checked": "2017-04-25T01:01:13.814Z", - "name": "run2_step1_outputprelim_out", - "user": 1, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 5 - }, - { - "fields": { - "description": "", - "filename": "sums_and_products.py", - "groups_allowed": [ - 1 - ], - "name": "sums_and_products", - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 1 - }, - { - "fields": { - "description": "", - "filename": "prelim_map.py", - "groups_allowed": [ - 1 - ], - "name": "prelim_map.py", - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 2 - }, - { - "fields": { - "description": "", - "filename": "__init__.py", - "groups_allowed": [ - 1 - ], - "name": "__init__.py", - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 3 - }, - { - "fields": { - "description": "", - "filename": "pssm_lib.py", - "groups_allowed": [ - 1 - ], - "name": "pssm_lib.py", - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 4 - }, - { - "fields": { - "description": "", - "filename": "sam2aln.py", - "groups_allowed": [ - 1 - ], - "name": "sam2aln.py", - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 5 - }, - { - "fields": { - "description": "", - "filename": "aln2counts.py", - "groups_allowed": [ - 1 - ], - "name": "aln2counts.py", - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 6 - }, - { - "fields": { - "description": "", - "filename": "miseq_logging.py", - "groups_allowed": [ - 1 - ], - "name": "miseq_logging.py", - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 7 - }, - { - "fields": { - "description": "", - "filename": "translation.py", - "groups_allowed": [ - 1 - ], - "name": "translation.py", - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 8 - }, - { - "fields": { - "description": "", - "filename": "sam_g2p.py", - "groups_allowed": [ - 1 - ], - "name": "sam_g2p.py", - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 9 - }, - { - "fields": { - "description": "", - "filename": "externals.py", - "groups_allowed": [ - 1 - ], - "name": "externals.py", - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 10 - }, - { - "fields": { - "description": "", - "filename": "remap.py", - "groups_allowed": [ - 1 - ], - "name": "remap.py", - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 11 - }, - { - "fields": { - "description": "", - "filename": "projects.json", - "groups_allowed": [ - 1 - ], - "name": "projects.json", - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 12 - }, - { - "fields": { - "description": "", - "filename": "coverage_plots.py", - "groups_allowed": [ - 1 - ], - "name": "coverage_plots.py", - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 13 - }, - { - "fields": { - "description": "", - "filename": "g2p.matrix", - "groups_allowed": [ - 1 - ], - "name": "g2p.matrix", - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 14 - }, - { - "fields": { - "description": "", - "filename": "project_config.py", - "groups_allowed": [ - 1 - ], - "name": "project_config.py", - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 15 - }, - { - "fields": { - "description": "", - "filename": "project_scoring.json", - "groups_allowed": [ - 1 - ], - "name": "project_scoring.json", - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 16 - }, - { - "fields": { - "description": "", - "filename": "g2p_fpr.txt", - "groups_allowed": [ - 1 - ], - "name": "g2p_fpr.txt", - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 17 - }, - { - "fields": { - "MD5_checksum": "e644faccd2f44dfae61a0fd9bf176134", - "coderesource": 1, - "content_file": "CodeResources/sums_and_products.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "", - "revision_name": "", - "revision_number": 1, - "revision_parent": null, - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "ae6ea4ec7835dd0ef8a788f4a44454e7", - "coderesource": 2, - "content_file": "CodeResources/prelim_map.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:18Z", - "revision_desc": "", - "revision_name": "", - "revision_number": 1, - "revision_parent": null, - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "d41d8cd98f00b204e9800998ecf8427e", - "coderesource": 3, - "content_file": "CodeResources/__init__.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:18Z", - "revision_desc": "", - "revision_name": "", - "revision_number": 1, - "revision_parent": null, - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 3 - }, - { - "fields": { - "MD5_checksum": "5c8fc3b09a3f99937139eb35e9d8bf63", - "coderesource": 4, - "content_file": "CodeResources/pssm_lib.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:18Z", - "revision_desc": "", - "revision_name": "", - "revision_number": 1, - "revision_parent": null, - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 4 - }, - { - "fields": { - "MD5_checksum": "712cb4c8483522de39b20bc0da1c0507", - "coderesource": 5, - "content_file": "CodeResources/sam2aln.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:18Z", - "revision_desc": "", - "revision_name": "", - "revision_number": 1, - "revision_parent": null, - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 5 - }, - { - "fields": { - "MD5_checksum": "66ca54f9f564d7de85b648cd093bcfe6", - "coderesource": 6, - "content_file": "CodeResources/aln2counts.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:18Z", - "revision_desc": "", - "revision_name": "", - "revision_number": 1, - "revision_parent": null, - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 6 - }, - { - "fields": { - "MD5_checksum": "ff879dc819f7c113144491e9e3c3b76f", - "coderesource": 7, - "content_file": "CodeResources/miseq_logging.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:18Z", - "revision_desc": "", - "revision_name": "", - "revision_number": 1, - "revision_parent": null, - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 7 - }, - { - "fields": { - "MD5_checksum": "9417aae918b0ac257ff3028b8d8d0409", - "coderesource": 8, - "content_file": "CodeResources/translation.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:18Z", - "revision_desc": "", - "revision_name": "", - "revision_number": 1, - "revision_parent": null, - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 8 - }, - { - "fields": { - "MD5_checksum": "6b58d46558be40d2d0e00cec697c26a6", - "coderesource": 9, - "content_file": "CodeResources/sam_g2p.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:18Z", - "revision_desc": "", - "revision_name": "", - "revision_number": 1, - "revision_parent": null, - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 9 - }, - { - "fields": { - "MD5_checksum": "cb07624aafbb43ba3c484872f1fb2e81", - "coderesource": 10, - "content_file": "CodeResources/externals.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:18Z", - "revision_desc": "", - "revision_name": "", - "revision_number": 1, - "revision_parent": null, - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 10 - }, - { - "fields": { - "MD5_checksum": "6e4e90524ca0c7e659b368d52dadb414", - "coderesource": 11, - "content_file": "CodeResources/remap.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:18Z", - "revision_desc": "", - "revision_name": "", - "revision_number": 1, - "revision_parent": null, - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 11 - }, - { - "fields": { - "MD5_checksum": "7efdd1cc9bc0ce2060a1df29aaad777e", - "coderesource": 12, - "content_file": "CodeResources/projects.json", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:18Z", - "revision_desc": "", - "revision_name": "", - "revision_number": 1, - "revision_parent": null, - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 12 - }, - { - "fields": { - "MD5_checksum": "45b1c9e45d75c2de32d257e018d19186", - "coderesource": 13, - "content_file": "CodeResources/coverage_plots.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:18Z", - "revision_desc": "", - "revision_name": "", - "revision_number": 1, - "revision_parent": null, - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 13 - }, - { - "fields": { - "MD5_checksum": "b4c8b3013b1b6822d056f5958115f50a", - "coderesource": 14, - "content_file": "CodeResources/g2p.matrix", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:18Z", - "revision_desc": "", - "revision_name": "", - "revision_number": 1, - "revision_parent": null, - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 14 - }, - { - "fields": { - "MD5_checksum": "b3781885c9844f797060b5d1233d6994", - "coderesource": 15, - "content_file": "CodeResources/project_config.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:18Z", - "revision_desc": "", - "revision_name": "", - "revision_number": 1, - "revision_parent": null, - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 15 - }, - { - "fields": { - "MD5_checksum": "e02cea1df30cecd5a763a860ae87e577", - "coderesource": 16, - "content_file": "CodeResources/project_scoring.json", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:18Z", - "revision_desc": "", - "revision_name": "", - "revision_number": 1, - "revision_parent": null, - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 16 - }, - { - "fields": { - "MD5_checksum": "246f62ac55cae327c98c82471941b344", - "coderesource": 17, - "content_file": "CodeResources/g2p_fpr.txt", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:18Z", - "revision_desc": "", - "revision_name": "", - "revision_number": 1, - "revision_parent": null, - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 17 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "reusable": 2, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 1 - }, - { - "fields": { - "driver": 2, - "family": 2, - "groups_allowed": [ - 1 - ], - "reusable": 2, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 3 - }, - { - "fields": { - "driver": 11, - "family": 3, - "groups_allowed": [ - 1 - ], - "reusable": 2, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 4 - }, - { - "fields": { - "driver": 5, - "family": 4, - "groups_allowed": [ - 1 - ], - "reusable": 2, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 5 - }, - { - "fields": { - "driver": 6, - "family": 5, - "groups_allowed": [ - 1 - ], - "reusable": 2, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 6 - }, - { - "fields": { - "driver": 9, - "family": 6, - "groups_allowed": [ - 1 - ], - "reusable": 2, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 7 - }, - { - "fields": { - "driver": 13, - "family": 7, - "groups_allowed": [ - 1 - ], - "reusable": 2, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 8 - }, - { - "fields": { - "description": "", - "groups_allowed": [ - 1 - ], - "name": "sums_and_products", - "user": 1, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 1 - }, - { - "fields": { - "description": "", - "groups_allowed": [ - 1 - ], - "name": "prelim", - "user": 1, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 2 - }, - { - "fields": { - "description": "", - "groups_allowed": [ - 1 - ], - "name": "remap", - "user": 1, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 3 - }, - { - "fields": { - "description": "", - "groups_allowed": [ - 1 - ], - "name": "sam2aln", - "user": 1, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 4 - }, - { - "fields": { - "description": "", - "groups_allowed": [ - 1 - ], - "name": "aln2counts", - "user": 1, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 5 - }, - { - "fields": { - "description": "", - "groups_allowed": [ - 1 - ], - "name": "sam_g2p", - "user": 1, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 6 - }, - { - "fields": { - "description": "", - "groups_allowed": [ - 1 - ], - "name": "coverage_plots", - "user": 1, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 7 - }, - { - "fields": { - "description": "", - "groups_allowed": [ - 1 - ], - "name": "sums and products", - "user": 1, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 1 - }, - { - "fields": { - "description": "", - "groups_allowed": [ - 1 - ], - "name": "MiCallDemo", - "user": 1, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 2 - }, - { - "fields": { - "family": 1, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 2 - }, - { - "fields": { - "family": 2, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 9 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "", - "revision_name": "first", - "user": 1, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 1 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "", - "revision_name": "sums and products", - "user": 1, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 2 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:18Z", - "revision_desc": "", - "revision_name": "first", - "user": 1, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 3 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:18Z", - "revision_desc": "", - "revision_name": "first", - "user": 1, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 4 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:18Z", - "revision_desc": "", - "revision_name": "first", - "user": 1, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:18Z", - "revision_desc": "", - "revision_name": "first", - "user": 1, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 6 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:19Z", - "revision_desc": "", - "revision_name": "first", - "user": 1, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 7 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:19Z", - "revision_desc": "", - "revision_name": "first", - "user": 1, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 8 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:19Z", - "revision_desc": "", - "revision_name": "MiCall Demo 2016-09", - "user": 1, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 9 - }, - { - "fields": { - "dataset": 2, - "end_time": "2000-01-01T00:00:14Z", - "execlog": 2, - "start_time": "2000-01-01T00:00:14Z", - "user": 1 - }, - "model": "datachecking.contentchecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 1, - "end_time": "2000-01-01T00:00:00Z", - "execlog": null, - "read_failed": false, - "runcomponent": 2, - "start_time": "2000-01-01T00:00:00Z", - "user": 1 - }, - "model": "datachecking.integritychecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "end_time": "2000-01-01T00:00:14Z", - "execlog": 2, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:14Z", - "user": 1 - }, - "model": "datachecking.integritychecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 3, - "end_time": "2000-01-01T00:00:21Z", - "execlog": null, - "read_failed": false, - "runcomponent": 10, - "start_time": "2000-01-01T00:00:21Z", - "user": 1 - }, - "model": "datachecking.integritychecklog", - "pk": 3 - }, - { - "fields": { - "dataset": 4, - "end_time": "2000-01-01T00:00:22Z", - "execlog": null, - "read_failed": false, - "runcomponent": 11, - "start_time": "2000-01-01T00:00:22Z", - "user": 1 - }, - "model": "datachecking.integritychecklog", - "pk": 4 - } -] \ No newline at end of file diff --git a/kive/portal/fixtures/em_sandbox_test_environment.json b/kive/portal/fixtures/em_sandbox_test_environment.json deleted file mode 100644 index 4efed6dca..000000000 --- a/kive/portal/fixtures/em_sandbox_test_environment.json +++ /dev/null @@ -1,2622 +0,0 @@ -[ - { - "fields": { - "datatype": 8, - "rule": "^[ACGTacgt]*$", - "ruletype": "regexp" - }, - "model": "metadata.basicconstraint", - "pk": 6 - }, - { - "fields": { - "datatype": 9, - "rule": "^[ACGUacgu]*$", - "ruletype": "regexp" - }, - "model": "metadata.basicconstraint", - "pk": 7 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "label", - "compounddatatype": 5, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 7 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "integer", - "compounddatatype": 5, - "datatype": 4 - }, - "model": "metadata.compounddatatypemember", - "pk": 8 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "float", - "compounddatatype": 5, - "datatype": 3 - }, - "model": "metadata.compounddatatypemember", - "pk": 9 - }, - { - "fields": { - "blankable": false, - "column_idx": 4, - "column_name": "bool", - "compounddatatype": 5, - "datatype": 2 - }, - "model": "metadata.compounddatatypemember", - "pk": 10 - }, - { - "fields": { - "blankable": false, - "column_idx": 5, - "column_name": "rna", - "compounddatatype": 5, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 11 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "label", - "compounddatatype": 6, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 12 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "label", - "compounddatatype": 7, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 13 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "PBMCseq", - "compounddatatype": 7, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 14 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "PLAseq", - "compounddatatype": 7, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 15 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "SeqToComplement", - "compounddatatype": 8, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 16 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "ComplementedSeq", - "compounddatatype": 9, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 17 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "SeqToComplement", - "compounddatatype": 10, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 18 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "ComplementedSeq", - "compounddatatype": 11, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 19 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "x", - "compounddatatype": 12, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 20 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "y", - "compounddatatype": 12, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 21 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "k", - "compounddatatype": 13, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 22 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "a", - "compounddatatype": 14, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 23 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "b", - "compounddatatype": 14, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 24 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "c", - "compounddatatype": 14, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 25 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "a^2", - "compounddatatype": 15, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 26 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "b^2", - "compounddatatype": 15, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 27 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "c^2", - "compounddatatype": 15, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 28 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "StrCol1", - "compounddatatype": 16, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 29 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "DNACol2", - "compounddatatype": 16, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 30 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "StrCol3", - "compounddatatype": 16, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 31 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "a", - "compounddatatype": 17, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 32 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "b", - "compounddatatype": 17, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 33 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "c", - "compounddatatype": 17, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 34 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "x", - "compounddatatype": 18, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 35 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "y", - "compounddatatype": 18, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 36 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "word", - "compounddatatype": 19, - "datatype": 10 - }, - "model": "metadata.compounddatatypemember", - "pk": 37 - }, - { - "fields": { - "dataset": 1, - "index": 1, - "run": 1 - }, - "model": "archive.runinput", - "pk": 1 - }, - { - "fields": { - "dataset": 3, - "index": 2, - "run": 1 - }, - "model": "archive.runinput", - "pk": 2 - }, - { - "fields": { - "dataset": 5, - "index": 3, - "run": 1 - }, - "model": "archive.runinput", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 14, - "dataset": 1, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 2, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 13, - "dataset": 3, - "num_rows": 13 - }, - "model": "librarian.datasetstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 13, - "dataset": 4, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 6, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 5 - }, - { - "fields": { - "compounddatatype": 14, - "dataset": 7, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 6 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 8, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 7 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 9, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 8 - }, - { - "fields": { - "compounddatatype": 13, - "dataset": 10, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 9 - }, - { - "fields": { - "compounddatatype": 14, - "dataset": 13, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 10 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 14, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 11 - }, - { - "fields": { - "compounddatatype": 17, - "dataset": 15, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 12 - }, - { - "fields": { - "compounddatatype": 18, - "dataset": 16, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 13 - }, - { - "fields": { - "compounddatatype": 18, - "dataset": 17, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 14 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 4, - "step_num": 1, - "transformation": 2, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 1 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 1, - "transformation": 1, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 2 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 2, - "transformation": 4, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 3 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 3, - "transformation": 3, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 4 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 1 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 2 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 3 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 4 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 5 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 6 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 7 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 8 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 9 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 10 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 11 - }, - { - "fields": { - "dest": 3, - "keep_output": false, - "pipelinestep": 1, - "source": 11, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 1 - }, - { - "fields": { - "dest": 4, - "keep_output": false, - "pipelinestep": 1, - "source": 12, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 2 - }, - { - "fields": { - "dest": 1, - "keep_output": false, - "pipelinestep": 2, - "source": 15, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 4 - }, - { - "fields": { - "dest": 11, - "keep_output": false, - "pipelinestep": 3, - "source": 13, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 5 - }, - { - "fields": { - "dest": 12, - "keep_output": false, - "pipelinestep": 3, - "source": 14, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 6 - }, - { - "fields": { - "dest": 7, - "keep_output": false, - "pipelinestep": 4, - "source": 2, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 7 - }, - { - "fields": { - "dest": 6, - "keep_output": false, - "pipelinestep": 4, - "source": 16, - "source_step": 2 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 8 - }, - { - "fields": { - "cable": 5, - "dest_pin": 21, - "source_pin": 23 - }, - "model": "pipeline.customcablewire", - "pk": 1 - }, - { - "fields": { - "cable": 5, - "dest_pin": 20, - "source_pin": 25 - }, - "model": "pipeline.customcablewire", - "pk": 2 - }, - { - "fields": { - "cable": 7, - "dest_pin": 21, - "source_pin": 20 - }, - "model": "pipeline.customcablewire", - "pk": 3 - }, - { - "fields": { - "cable": 7, - "dest_pin": 20, - "source_pin": 21 - }, - "model": "pipeline.customcablewire", - "pk": 4 - }, - { - "fields": { - "cable": 9, - "dest_pin": 21, - "source_pin": 24 - }, - "model": "pipeline.customcablewire", - "pk": 5 - }, - { - "fields": { - "cable": 9, - "dest_pin": 20, - "source_pin": 25 - }, - "model": "pipeline.customcablewire", - "pk": 6 - }, - { - "fields": { - "output_cdt": 14, - "output_idx": 1, - "output_name": "D1_out", - "pipeline": 4, - "source": 5, - "source_step": 1 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 3 - }, - { - "fields": { - "output_cdt": 12, - "output_idx": 1, - "output_name": "E1_out", - "pipeline": 5, - "source": 16, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 9 - }, - { - "fields": { - "output_cdt": 13, - "output_idx": 2, - "output_name": "E2_out", - "pipeline": 5, - "source": 8, - "source_step": 3 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 10 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 3, - "output_name": "E3_rawout", - "pipeline": 5, - "source": 10, - "source_step": 3 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 11 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 1 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 2 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 3 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 4 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 5 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 6 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 7 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 8 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 9 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 10 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 11 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 12 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 13 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 14 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 15 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 16 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 17 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 18 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 19 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 20 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 21 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 22 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 23 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 24 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 25 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 2 - }, - "model": "transformation.xputstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 3 - }, - "model": "transformation.xputstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 4 - }, - "model": "transformation.xputstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": 5, - "min_row": null, - "transf_xput": 5 - }, - "model": "transformation.xputstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": null, - "min_row": null, - "transf_xput": 6 - }, - "model": "transformation.xputstructure", - "pk": 5 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 7 - }, - "model": "transformation.xputstructure", - "pk": 6 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 8 - }, - "model": "transformation.xputstructure", - "pk": 7 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 11 - }, - "model": "transformation.xputstructure", - "pk": 8 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 12 - }, - "model": "transformation.xputstructure", - "pk": 9 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": null, - "min_row": null, - "transf_xput": 13 - }, - "model": "transformation.xputstructure", - "pk": 10 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": 10, - "transf_xput": 14 - }, - "model": "transformation.xputstructure", - "pk": 11 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": 5, - "min_row": null, - "transf_xput": 16 - }, - "model": "transformation.xputstructure", - "pk": 12 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 18 - }, - "model": "transformation.xputstructure", - "pk": 13 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": 5, - "min_row": null, - "transf_xput": 19 - }, - "model": "transformation.xputstructure", - "pk": 14 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 20 - }, - "model": "transformation.xputstructure", - "pk": 15 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 21 - }, - "model": "transformation.xputstructure", - "pk": 16 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 22 - }, - "model": "transformation.xputstructure", - "pk": 17 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 23 - }, - "model": "transformation.xputstructure", - "pk": 18 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "A1_rawin", - "transformation": 1 - }, - "model": "transformation.transformationinput", - "pk": 1 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "B1_in", - "transformation": 2 - }, - "model": "transformation.transformationinput", - "pk": 3 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "B2_in", - "transformation": 2 - }, - "model": "transformation.transformationinput", - "pk": 4 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "C1_in", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 6 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "C2_in", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 7 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "D1_in", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 11 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "D2_in", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 12 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "E1_in", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 13 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "E2_in", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 14 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "E3_rawin", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 15 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "strings", - "transformation": 6 - }, - "model": "transformation.transformationinput", - "pk": 20 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "strings", - "transformation": 7 - }, - "model": "transformation.transformationinput", - "pk": 22 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "raw", - "transformation": 8 - }, - "model": "transformation.transformationinput", - "pk": 24 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "A1_out", - "transformation": 1 - }, - "model": "transformation.transformationoutput", - "pk": 2 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "B1_out", - "transformation": 2 - }, - "model": "transformation.transformationoutput", - "pk": 5 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "C1_out", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 8 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "C2_rawout", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 9 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "C3_rawout", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 10 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "D1_out", - "transformation": 4 - }, - "model": "transformation.transformationoutput", - "pk": 16 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "E3_rawout", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 17 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "E2_out", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 18 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "E1_out", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 19 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "same_strings", - "transformation": 6 - }, - "model": "transformation.transformationoutput", - "pk": 21 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "untouched_strings", - "transformation": 7 - }, - "model": "transformation.transformationoutput", - "pk": 23 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "same_raw", - "transformation": 8 - }, - "model": "transformation.transformationoutput", - "pk": 25 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:00Z", - "email": "lennon@thebeatles.com", - "first_name": "", - "groups": [ - 1 - ], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$36000$07QF8yio6Edl$uDACUL4+PP6fx3VLjPi+i4rEq3N69ffBzoXHDd1r7mA=", - "user_permissions": [], - "username": "john" - }, - "model": "auth.user", - "pk": 2 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:00Z", - "email": "starr@thebeatles.com", - "first_name": "", - "groups": [], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$36000$mvPKvcEvVPq7$8iyztccS8MiFFZMLjsETkvzTwD42k4PPtTYvhMibMhA=", - "user_permissions": [], - "username": "ringo" - }, - "model": "auth.user", - "pk": 3 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:05Z", - "email": "bob@talabs.com", - "first_name": "", - "groups": [ - 1 - ], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$36000$JrRWq0YEoawQ$a0Gclr7en2jD28+9rJ0woiyKVptPPge2UVg7IJ9L8rk=", - "user_permissions": [], - "username": "bob" - }, - "model": "auth.user", - "pk": 4 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:00Z", - "description": "String consisting of ACGTacgt", - "groups_allowed": [ - 1 - ], - "name": "DNANucSeq", - "prototype": null, - "restricts": [ - 1 - ], - "user": 2, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 8 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:00Z", - "description": "String consisting of ACGUacgu", - "groups_allowed": [ - 1 - ], - "name": "RNANucSeq", - "prototype": null, - "restricts": [ - 1 - ], - "user": 2, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 9 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:05Z", - "description": "sequences of ASCII characters", - "groups_allowed": [ - 1 - ], - "name": "my_string", - "prototype": null, - "restricts": [ - 1 - ], - "user": 4, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 10 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [], - "name": "", - "user": 2, - "users_allowed": [ - 3 - ] - }, - "model": "metadata.compounddatatype", - "pk": 6 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 7 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 8 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 9 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 10 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 11 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 12 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 13 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 14 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 15 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 16 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 17 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 18 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 4, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 19 - }, - { - "fields": { - "_runstate": 1, - "description": "", - "end_time": null, - "groups_allowed": [ - 1 - ], - "name": "pE_run", - "parent_runstep": null, - "paused_by": null, - "pipeline": 5, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "", - "start_time": null, - "stopped_by": null, - "time_queued": "2018-06-01T00:04:40.590Z", - "user": 2, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "ab7afe2f453d27fd168641a1f9271f26", - "_redacted": false, - "dataset_file": "Datasets/2018_05/step_0_triplet.csv", - "date_created": "2000-01-01T00:00:04Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2018-06-01T00:04:38.103Z", - "name": "triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "02d5b66858d003b81ae3dc712b74e85c", - "_redacted": false, - "dataset_file": "Datasets/2018_05/doublet_cdt.csv", - "date_created": "2000-01-01T00:00:04Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2018-06-01T00:04:38.196Z", - "name": "doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "29feb269edcac613d6eea432c14640bd", - "_redacted": false, - "dataset_file": "Datasets/2018_05/singlet_cdt_large.csv", - "date_created": "2000-01-01T00:00:04Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2018-06-01T00:04:38.272Z", - "name": "singlet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 3 - }, - { - "fields": { - "MD5_checksum": "00fd50bfe7ac63e2165a442f4101e05c", - "_redacted": false, - "dataset_file": "Datasets/2018_05/step_0_singlet.csv", - "date_created": "2000-01-01T00:00:04Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2018-06-01T00:04:38.333Z", - "name": "singlet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 4 - }, - { - "fields": { - "MD5_checksum": "7dc85e11b5c02e434af5bd3b3da9938e", - "_redacted": false, - "dataset_file": "Datasets/2018_05/step_0_raw.fasta", - "date_created": "2000-01-01T00:00:04Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2018-06-01T00:04:38.378Z", - "name": "raw_DS", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 5 - }, - { - "fields": { - "MD5_checksum": "542676b23e121d16db8d41ccdae65fd1", - "_redacted": false, - "dataset_file": "", - "date_created": "2000-01-01T00:00:04Z", - "description": "", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2018-06-01T00:04:38.410Z", - "name": "D1_in_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 6 - }, - { - "fields": { - "MD5_checksum": "56f9e5585b9d699829e2fda931d944f8", - "_redacted": false, - "dataset_file": "Datasets/2018_05/C1_in_triplet.csv", - "date_created": "2000-01-01T00:00:04Z", - "description": "triplet 3 rows", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2018-06-01T00:04:38.474Z", - "name": "C1_in_triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 7 - }, - { - "fields": { - "MD5_checksum": "10a4bd15d1300fcaf5a3bdad54ef1d28", - "_redacted": false, - "dataset_file": "", - "date_created": "2000-01-01T00:00:04Z", - "description": "", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2018-06-01T00:04:38.551Z", - "name": "C2_in_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 8 - }, - { - "fields": { - "MD5_checksum": "10a4bd15d1300fcaf5a3bdad54ef1d28", - "_redacted": false, - "dataset_file": "Datasets/2018_05/E11_32_output.csv", - "date_created": "2000-01-01T00:00:04Z", - "description": "result of E11_32 fed by doublet_cdt.csv", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2018-06-01T00:04:38.640Z", - "name": "E11_32 output doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 9 - }, - { - "fields": { - "MD5_checksum": "00fd50bfe7ac63e2165a442f4101e05c", - "_redacted": false, - "dataset_file": "Datasets/2018_05/step_0_singlet_iw1yqwb.csv", - "date_created": "2000-01-01T00:00:04Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2018-06-01T00:04:38.730Z", - "name": "raw", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 10 - }, - { - "fields": { - "MD5_checksum": "7dc85e11b5c02e434af5bd3b3da9938e", - "_redacted": false, - "dataset_file": "Datasets/2018_05/step_0_raw_pAslszd.fasta", - "date_created": "2000-01-01T00:00:04Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2018-06-01T00:04:38.795Z", - "name": "C2_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 11 - }, - { - "fields": { - "MD5_checksum": "7dc85e11b5c02e434af5bd3b3da9938e", - "_redacted": false, - "dataset_file": "Datasets/2018_05/step_0_raw_XjjIXiN.fasta", - "date_created": "2000-01-01T00:00:04Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2018-06-01T00:04:38.825Z", - "name": "C3_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 12 - }, - { - "fields": { - "MD5_checksum": "56f9e5585b9d699829e2fda931d944f8", - "_redacted": false, - "dataset_file": "Datasets/2018_05/step_0_triplet_3_rows.csv", - "date_created": "2000-01-01T00:00:04Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2018-06-01T00:04:38.865Z", - "name": "triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 13 - }, - { - "fields": { - "MD5_checksum": "83957f8aea1c75da9f7fbf9bc90da979", - "_redacted": false, - "dataset_file": "Datasets/2018_05/doublet_remuxed_from_t3r.csv", - "date_created": "2000-01-01T00:00:05Z", - "description": "doublet remuxed from triplet", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2018-06-01T00:04:38.980Z", - "name": "E1_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 14 - }, - { - "fields": { - "MD5_checksum": "d83866e4c23cd9503aa2f9fc565502b5", - "_redacted": false, - "dataset_file": "Datasets/2018_05/DNA_triplet.csv", - "date_created": "2000-01-01T00:00:05Z", - "description": "DNA triplet data", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2018-06-01T00:04:39.057Z", - "name": "DNA_triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 15 - }, - { - "fields": { - "MD5_checksum": "9eab0dfed5fcc89bd581f61995816ef0", - "_redacted": false, - "dataset_file": "Datasets/2018_05/E01_21_DNA_doublet.csv", - "date_created": "2000-01-01T00:00:05Z", - "description": "DNA doublet data coming from DNA_triplet.csv but remultiplexed according to cable E01_21", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2018-06-01T00:04:39.200Z", - "name": "E01_21_DNA_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 16 - }, - { - "fields": { - "MD5_checksum": "9b55b9431da6dba74ea61edf8e2ac044", - "_redacted": false, - "dataset_file": "Datasets/2018_05/E21_41_DNA_doublet.csv", - "date_created": "2000-01-01T00:00:05Z", - "description": "DNA doublet data coming from DNA_triplet.csv but remultiplexed according to cable E21_41", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2018-06-01T00:04:39.343Z", - "name": "E21_41_DNA_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 17 - }, - { - "fields": { - "description": "Just a CR", - "filename": "generic_script.py", - "groups_allowed": [ - 1 - ], - "name": "genericCR", - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 1 - }, - { - "fields": { - "description": "a script to do nothing", - "filename": "noop.sh", - "groups_allowed": [ - 1 - ], - "name": "noop", - "user": 4, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "fe9e542c616de5d823fe100fc293acc1", - "coderesource": 1, - "content_file": "samplecode/generic_script_A9Zzeyi.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "desc", - "revision_name": "v1", - "revision_number": 1, - "revision_parent": null, - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "d218c00725d473639f347d456de381d8", - "coderesource": 2, - "content_file": "CodeResources/blaname", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:05Z", - "revision_desc": "first version", - "revision_name": "1", - "revision_number": 1, - "revision_parent": null, - "user": 4, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 2 - }, - { - "fields": { - "docker_image": null, - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "memory": 6000, - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 1 - }, - { - "fields": { - "docker_image": null, - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "memory": 6000, - "reusable": 1, - "revision_number": 2, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 2 - }, - { - "fields": { - "docker_image": null, - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "memory": 6000, - "reusable": 1, - "revision_number": 3, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 3 - }, - { - "fields": { - "docker_image": null, - "driver": 2, - "family": 2, - "groups_allowed": [ - 1 - ], - "memory": 6000, - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 6 - }, - { - "fields": { - "docker_image": null, - "driver": 2, - "family": 3, - "groups_allowed": [ - 1 - ], - "memory": 6000, - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 7 - }, - { - "fields": { - "docker_image": null, - "driver": 2, - "family": 4, - "groups_allowed": [ - 1 - ], - "memory": 6000, - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 8 - }, - { - "fields": { - "description": "Holds methods A/B/C", - "groups_allowed": [ - 1 - ], - "name": "method_family", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 1 - }, - { - "fields": { - "description": "a method to do nothing to strings", - "groups_allowed": [ - 1 - ], - "name": "string noop", - "user": 4, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 2 - }, - { - "fields": { - "description": "a TOTALLY DIFFERENT method that TOTALLY does SOMETHING to strings by leaving them alone", - "groups_allowed": [ - 1 - ], - "name": "string trivial", - "user": 4, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 3 - }, - { - "fields": { - "description": "do nothing to raw data", - "groups_allowed": [ - 1 - ], - "name": "raw noop", - "user": 4, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 4 - }, - { - "fields": { - "description": "PF desc", - "groups_allowed": [ - 1 - ], - "name": "Pipeline_family", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 1 - }, - { - "fields": { - "family": 1, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 4 - }, - { - "fields": { - "family": 1, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 2, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "A_desc", - "revision_name": "mA_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 1 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "B_desc", - "revision_name": "mB_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 2 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "C_desc", - "revision_name": "mC_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 3 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "D", - "revision_name": "pD_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 4 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:03Z", - "revision_desc": "E", - "revision_name": "pE_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:05Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 4, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 6 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:06Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 4, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 7 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:06Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 4, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 8 - }, - { - "fields": { - "dataset": 1, - "end_time": "2000-01-01T00:00:04Z", - "execlog": null, - "start_time": "2000-01-01T00:00:04Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "end_time": "2000-01-01T00:00:04Z", - "execlog": null, - "start_time": "2000-01-01T00:00:04Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 3, - "end_time": "2000-01-01T00:00:04Z", - "execlog": null, - "start_time": "2000-01-01T00:00:04Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 3 - }, - { - "fields": { - "dataset": 4, - "end_time": "2000-01-01T00:00:04Z", - "execlog": null, - "start_time": "2000-01-01T00:00:04Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 4 - }, - { - "fields": { - "dataset": 6, - "end_time": "2000-01-01T00:00:04Z", - "execlog": null, - "start_time": "2000-01-01T00:00:04Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 5 - }, - { - "fields": { - "dataset": 7, - "end_time": "2000-01-01T00:00:04Z", - "execlog": null, - "start_time": "2000-01-01T00:00:04Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 6 - }, - { - "fields": { - "dataset": 8, - "end_time": "2000-01-01T00:00:04Z", - "execlog": null, - "start_time": "2000-01-01T00:00:04Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 7 - }, - { - "fields": { - "dataset": 9, - "end_time": "2000-01-01T00:00:04Z", - "execlog": null, - "start_time": "2000-01-01T00:00:04Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 8 - }, - { - "fields": { - "dataset": 10, - "end_time": "2000-01-01T00:00:04Z", - "execlog": null, - "start_time": "2000-01-01T00:00:04Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 9 - }, - { - "fields": { - "dataset": 13, - "end_time": "2000-01-01T00:00:05Z", - "execlog": null, - "start_time": "2000-01-01T00:00:04Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 10 - }, - { - "fields": { - "dataset": 14, - "end_time": "2000-01-01T00:00:05Z", - "execlog": null, - "start_time": "2000-01-01T00:00:05Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 11 - }, - { - "fields": { - "dataset": 15, - "end_time": "2000-01-01T00:00:05Z", - "execlog": null, - "start_time": "2000-01-01T00:00:05Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 12 - }, - { - "fields": { - "dataset": 16, - "end_time": "2000-01-01T00:00:05Z", - "execlog": null, - "start_time": "2000-01-01T00:00:05Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 13 - }, - { - "fields": { - "dataset": 17, - "end_time": "2000-01-01T00:00:05Z", - "execlog": null, - "start_time": "2000-01-01T00:00:05Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 14 - } -] \ No newline at end of file diff --git a/kive/portal/fixtures/execute_discarded_intermediate_tests_rm.json b/kive/portal/fixtures/execute_discarded_intermediate_tests_rm.json deleted file mode 100644 index 091bb8eb1..000000000 --- a/kive/portal/fixtures/execute_discarded_intermediate_tests_rm.json +++ /dev/null @@ -1,2256 +0,0 @@ -[ - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "word", - "compounddatatype": 5, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 7 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "header", - "compounddatatype": 6, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 8 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "sequence", - "compounddatatype": 6, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 9 - }, - { - "fields": { - "dataset": 1, - "index": 1, - "run": 1 - }, - "model": "archive.runinput", - "pk": 1 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:17Z", - "execrecord": 2, - "reused": false, - "start_time": "2000-01-01T00:00:04Z" - }, - "model": "archive.runcomponent", - "pk": 1 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:31Z", - "execrecord": 4, - "reused": false, - "start_time": "2000-01-01T00:00:18Z" - }, - "model": "archive.runcomponent", - "pk": 2 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:05Z", - "execrecord": 1, - "reused": false, - "start_time": "2000-01-01T00:00:04Z" - }, - "model": "archive.runcomponent", - "pk": 3 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:19Z", - "execrecord": 3, - "reused": false, - "start_time": "2000-01-01T00:00:18Z" - }, - "model": "archive.runcomponent", - "pk": 4 - }, - { - "fields": { - "pipelinestep": 6, - "run": 1 - }, - "model": "archive.runstep", - "pk": 1 - }, - { - "fields": { - "pipelinestep": 7, - "run": 1 - }, - "model": "archive.runstep", - "pk": 2 - }, - { - "fields": { - "PSIC": 10, - "dest_runstep": 1 - }, - "model": "archive.runsic", - "pk": 3 - }, - { - "fields": { - "PSIC": 11, - "dest_runstep": 2 - }, - "model": "archive.runsic", - "pk": 4 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:05Z", - "invoking_record": 3, - "record": 3, - "start_time": "2000-01-01T00:00:05Z" - }, - "model": "archive.execlog", - "pk": 1 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:05Z", - "invoking_record": 1, - "record": 1, - "start_time": "2000-01-01T00:00:05Z" - }, - "model": "archive.execlog", - "pk": 2 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:18Z", - "invoking_record": 4, - "record": 4, - "start_time": "2000-01-01T00:00:18Z" - }, - "model": "archive.execlog", - "pk": 3 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:19Z", - "invoking_record": 2, - "record": 2, - "start_time": "2000-01-01T00:00:19Z" - }, - "model": "archive.execlog", - "pk": 4 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step1_stderr_slurmIDJ_nodeN.txt", - "error_redacted": false, - "execlog": 2, - "install_failed": false, - "output_log": "Logs/step1_stdout_slurmIDJ_nodeN.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 1 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step2_stderr_slurmIDJ_nodeN.txt", - "error_redacted": false, - "execlog": 4, - "install_failed": false, - "output_log": "Logs/step2_stdout_slurmIDJ_nodeN.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 6, - "dataset": 1, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 6, - "dataset": 2, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 6, - "dataset": 3, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 3 - }, - { - "fields": { - "generator": 1 - }, - "model": "librarian.execrecord", - "pk": 1 - }, - { - "fields": { - "generator": 2 - }, - "model": "librarian.execrecord", - "pk": 2 - }, - { - "fields": { - "generator": 3 - }, - "model": "librarian.execrecord", - "pk": 3 - }, - { - "fields": { - "generator": 4 - }, - "model": "librarian.execrecord", - "pk": 4 - }, - { - "fields": { - "dataset": 1, - "execrecord": 1, - "generic_input": 22 - }, - "model": "librarian.execrecordin", - "pk": 1 - }, - { - "fields": { - "dataset": 1, - "execrecord": 2, - "generic_input": 9 - }, - "model": "librarian.execrecordin", - "pk": 2 - }, - { - "fields": { - "dataset": 2, - "execrecord": 3, - "generic_input": 10 - }, - "model": "librarian.execrecordin", - "pk": 3 - }, - { - "fields": { - "dataset": 2, - "execrecord": 4, - "generic_input": 7 - }, - "model": "librarian.execrecordin", - "pk": 4 - }, - { - "fields": { - "dataset": 1, - "execrecord": 1, - "generic_output": 9 - }, - "model": "librarian.execrecordout", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "execrecord": 2, - "generic_output": 10 - }, - "model": "librarian.execrecordout", - "pk": 2 - }, - { - "fields": { - "dataset": 2, - "execrecord": 3, - "generic_output": 7 - }, - "model": "librarian.execrecordout", - "pk": 3 - }, - { - "fields": { - "dataset": 3, - "execrecord": 4, - "generic_output": 8 - }, - "model": "librarian.execrecordout", - "pk": 4 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 4, - "step_num": 1, - "transformation": 7, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 1 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 1, - "transformation": 8, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 2 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 6, - "step_num": 1, - "transformation": 8, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 3 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 6, - "step_num": 2, - "transformation": 7, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 4 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [ - 8 - ], - "pipeline": 9, - "step_num": 1, - "transformation": 7, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 5 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [ - 10 - ], - "pipeline": 10, - "step_num": 1, - "transformation": 8, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 6 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [ - 8 - ], - "pipeline": 10, - "step_num": 2, - "transformation": 7, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 7 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [ - 10 - ], - "pipeline": 11, - "step_num": 1, - "transformation": 8, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 8 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 11, - "step_num": 2, - "transformation": 7, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 9 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 13, - "step_num": 1, - "transformation": 8, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 10 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 13, - "step_num": 2, - "transformation": 12, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 11 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 1 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 2 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 3 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 4 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 5 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 6 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 7 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 8 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 10 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 11 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 13 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 14 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 15 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 16 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 17 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 18 - }, - { - "fields": { - "dest": 7, - "keep_output": false, - "pipelinestep": 1, - "source": 11, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 1 - }, - { - "fields": { - "dest": 9, - "keep_output": false, - "pipelinestep": 2, - "source": 14, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 3 - }, - { - "fields": { - "dest": 9, - "keep_output": false, - "pipelinestep": 3, - "source": 17, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 5 - }, - { - "fields": { - "dest": 7, - "keep_output": false, - "pipelinestep": 4, - "source": 10, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 6 - }, - { - "fields": { - "dest": 7, - "keep_output": false, - "pipelinestep": 5, - "source": 20, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 8 - }, - { - "fields": { - "dest": 9, - "keep_output": false, - "pipelinestep": 6, - "source": 22, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 10 - }, - { - "fields": { - "dest": 7, - "keep_output": false, - "pipelinestep": 7, - "source": 10, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 11 - }, - { - "fields": { - "dest": 9, - "keep_output": false, - "pipelinestep": 8, - "source": 24, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 13 - }, - { - "fields": { - "dest": 7, - "keep_output": false, - "pipelinestep": 9, - "source": 10, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 14 - }, - { - "fields": { - "dest": 9, - "keep_output": false, - "pipelinestep": 10, - "source": 29, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 16 - }, - { - "fields": { - "dest": 27, - "keep_output": false, - "pipelinestep": 11, - "source": 10, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 17 - }, - { - "fields": { - "output_cdt": 6, - "output_idx": 1, - "output_name": "complemented_lab_data", - "pipeline": 4, - "source": 8, - "source_step": 1 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 2 - }, - { - "fields": { - "output_cdt": 6, - "output_idx": 1, - "output_name": "reversed_lab_data", - "pipeline": 5, - "source": 10, - "source_step": 1 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 4 - }, - { - "fields": { - "output_cdt": 6, - "output_idx": 1, - "output_name": "reverse_and_complemented_lab_data", - "pipeline": 6, - "source": 8, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 7 - }, - { - "fields": { - "output_cdt": 6, - "output_idx": 1, - "output_name": "revcomped_lab_data", - "pipeline": 11, - "source": 8, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 15 - }, - { - "fields": { - "output_cdt": 6, - "output_idx": 1, - "output_name": "RNAd_lab_data", - "pipeline": 13, - "source": 28, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 18 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 1 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 2 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 3 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 4 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 5 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 6 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 7 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 8 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 9 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 10 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 11 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 13 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 14 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 16 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 17 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 19 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 20 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 22 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 24 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 26 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 27 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 28 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 29 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 31 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 1 - }, - "model": "transformation.xputstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 2 - }, - "model": "transformation.xputstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 3 - }, - "model": "transformation.xputstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 4 - }, - "model": "transformation.xputstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 7 - }, - "model": "transformation.xputstructure", - "pk": 5 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 8 - }, - "model": "transformation.xputstructure", - "pk": 6 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 9 - }, - "model": "transformation.xputstructure", - "pk": 7 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 10 - }, - "model": "transformation.xputstructure", - "pk": 8 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 11 - }, - "model": "transformation.xputstructure", - "pk": 9 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 13 - }, - "model": "transformation.xputstructure", - "pk": 11 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 14 - }, - "model": "transformation.xputstructure", - "pk": 12 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 16 - }, - "model": "transformation.xputstructure", - "pk": 14 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 17 - }, - "model": "transformation.xputstructure", - "pk": 15 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 19 - }, - "model": "transformation.xputstructure", - "pk": 17 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 20 - }, - "model": "transformation.xputstructure", - "pk": 18 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 22 - }, - "model": "transformation.xputstructure", - "pk": 20 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 24 - }, - "model": "transformation.xputstructure", - "pk": 22 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 26 - }, - "model": "transformation.xputstructure", - "pk": 24 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 27 - }, - "model": "transformation.xputstructure", - "pk": 25 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 28 - }, - "model": "transformation.xputstructure", - "pk": 26 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 29 - }, - "model": "transformation.xputstructure", - "pk": 27 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 31 - }, - "model": "transformation.xputstructure", - "pk": 29 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "strings", - "transformation": 1 - }, - "model": "transformation.transformationinput", - "pk": 1 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "strings", - "transformation": 2 - }, - "model": "transformation.transformationinput", - "pk": 3 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "raw", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 5 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "DNA_to_complement", - "transformation": 7 - }, - "model": "transformation.transformationinput", - "pk": 7 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "DNA_to_reverse", - "transformation": 8 - }, - "model": "transformation.transformationinput", - "pk": 9 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "lab_data", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 11 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "lab_data", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 14 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "lab_data", - "transformation": 6 - }, - "model": "transformation.transformationinput", - "pk": 17 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "lab_data", - "transformation": 9 - }, - "model": "transformation.transformationinput", - "pk": 20 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "lab_data", - "transformation": 10 - }, - "model": "transformation.transformationinput", - "pk": 22 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "lab_data", - "transformation": 11 - }, - "model": "transformation.transformationinput", - "pk": 24 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "DNA_to_convert", - "transformation": 12 - }, - "model": "transformation.transformationinput", - "pk": 27 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "lab_data", - "transformation": 13 - }, - "model": "transformation.transformationinput", - "pk": 29 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "same_strings", - "transformation": 1 - }, - "model": "transformation.transformationoutput", - "pk": 2 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "untouched_strings", - "transformation": 2 - }, - "model": "transformation.transformationoutput", - "pk": 4 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "same_raw", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 6 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "complemented_DNA", - "transformation": 7 - }, - "model": "transformation.transformationoutput", - "pk": 8 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "reversed_DNA", - "transformation": 8 - }, - "model": "transformation.transformationoutput", - "pk": 10 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "complemented_lab_data", - "transformation": 4 - }, - "model": "transformation.transformationoutput", - "pk": 13 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "reversed_lab_data", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 16 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "reverse_and_complemented_lab_data", - "transformation": 6 - }, - "model": "transformation.transformationoutput", - "pk": 19 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "revcomped_lab_data", - "transformation": 11 - }, - "model": "transformation.transformationoutput", - "pk": 26 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "RNA", - "transformation": 12 - }, - "model": "transformation.transformationoutput", - "pk": 28 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "RNAd_lab_data", - "transformation": 13 - }, - "model": "transformation.transformationoutput", - "pk": 31 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:00Z", - "email": "bob@talabs.com", - "first_name": "", - "groups": [ - 1 - ], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$RXy17pdrf2oz$eczge2IsR1W5TS8RihDXJoQYr2ov7is7kobzmW8y2yU=", - "user_permissions": [], - "username": "bob" - }, - "model": "auth.user", - "pk": 2 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:00Z", - "email": "alice@talabs.com", - "first_name": "", - "groups": [ - 1 - ], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$0XwWVfZpt3LF$Fw9xG4j8ede56IEPKLz7kmtp55vEFhTlhmUB/cfdPz8=", - "user_permissions": [], - "username": "alice" - }, - "model": "auth.user", - "pk": 3 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:00Z", - "description": "sequences of ASCII characters", - "groups_allowed": [ - 1 - ], - "name": "my_string", - "prototype": null, - "restricts": [ - 1 - ], - "user": 2, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 8 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:00Z", - "description": "sequences of ATCG", - "groups_allowed": [ - 1 - ], - "name": "DNA", - "prototype": null, - "restricts": [ - 1 - ], - "user": 3, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 9 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 3, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 6 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:33Z", - "groups_allowed": [], - "name": "", - "parent_runstep": null, - "paused_by": null, - "pipeline": 10, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "/home/rliang/Documents/Kive/media/Sandboxes/useralice_run1__rk_ts", - "start_time": "2000-01-01T00:00:04Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:58:52.820Z", - "user": 3, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "0082bc75b97d81a1b92adaa894fef1bd", - "_redacted": false, - "dataset_file": "Datasets/2017_04/tmpAKYF_8", - "date_created": "2000-01-01T00:00:01Z", - "description": "data from the lab", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [], - "last_time_checked": "2017-04-25T00:58:50.062Z", - "name": "lab data", - "user": 3, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "c0d4cb49470a30a88ea5d4cf0c3324c9", - "_redacted": false, - "dataset_file": "", - "date_created": "2000-01-01T00:00:16Z", - "description": "Generated data from a run of pipeline \"DNA revcomp:2 (2)\" started at 2017-04-25 00:58:52.968542+00:00 by alice\nrun: 1\nuser: alice\nstep: 1\noutput: reversed_DNA", - "external_path": "", - "externalfiledirectory": null, - "file_source": 1, - "groups_allowed": [], - "last_time_checked": "2017-04-25T00:59:04.382Z", - "name": "run1_step1_outputreversed_DNA", - "user": 3, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "0082bc75b97d81a1b92adaa894fef1bd", - "_redacted": false, - "dataset_file": "", - "date_created": "2000-01-01T00:00:30Z", - "description": "Generated data from a run of pipeline \"DNA revcomp:2 (2)\" started at 2017-04-25 00:58:52.968542+00:00 by alice\nrun: 1\nuser: alice\nstep: 2\noutput: complemented_DNA", - "external_path": "", - "externalfiledirectory": null, - "file_source": 2, - "groups_allowed": [], - "last_time_checked": "2017-04-25T00:59:19.225Z", - "name": "run1_step2_outputcomplemented_DNA", - "user": 3, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 3 - }, - { - "fields": { - "description": "a script to do nothing", - "filename": "noop.sh", - "groups_allowed": [ - 1 - ], - "name": "noop", - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 1 - }, - { - "fields": { - "description": "a script to complement DNA", - "filename": "complement.sh", - "groups_allowed": [ - 1 - ], - "name": "DNA complement", - "user": 3, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 2 - }, - { - "fields": { - "description": "a script to reverse DNA", - "filename": "reverse.sh", - "groups_allowed": [ - 1 - ], - "name": "DNA reverse", - "user": 3, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 3 - }, - { - "fields": { - "description": "a script to reverse DNA", - "filename": "DNA2RNA.sh", - "groups_allowed": [ - 1 - ], - "name": "DNA to RNA", - "user": 3, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 4 - }, - { - "fields": { - "MD5_checksum": "d218c00725d473639f347d456de381d8", - "coderesource": 1, - "content_file": "CodeResources/fdopen", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "1", - "revision_number": 1, - "revision_parent": null, - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "5699120b018a7932c07be0412483c622", - "coderesource": 2, - "content_file": "CodeResources/fdopen_Tid5Mnk", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "1", - "revision_number": 1, - "revision_parent": null, - "user": 3, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "196144423c5bb98d59d78c729ce842f4", - "coderesource": 3, - "content_file": "CodeResources/fdopen_Y4ejrBL", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "1", - "revision_number": 1, - "revision_parent": null, - "user": 3, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 3 - }, - { - "fields": { - "MD5_checksum": "30e9a1ac1dad0c134d41bf826a53ba68", - "coderesource": 4, - "content_file": "CodeResources/fdopen_IOyU8CE", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:03Z", - "revision_desc": "first version", - "revision_name": "1", - "revision_number": 1, - "revision_parent": null, - "user": 3, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 4 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 1 - }, - { - "fields": { - "driver": 1, - "family": 2, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 2 - }, - { - "fields": { - "driver": 1, - "family": 3, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 3 - }, - { - "fields": { - "driver": 2, - "family": 4, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 7 - }, - { - "fields": { - "driver": 2, - "family": 5, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 8 - }, - { - "fields": { - "driver": 4, - "family": 6, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 12 - }, - { - "fields": { - "description": "a method to do nothing to strings", - "groups_allowed": [ - 1 - ], - "name": "string noop", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 1 - }, - { - "fields": { - "description": "a TOTALLY DIFFERENT method that TOTALLY does SOMETHING to strings by leaving them alone", - "groups_allowed": [ - 1 - ], - "name": "string trivial", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 2 - }, - { - "fields": { - "description": "do nothing to raw data", - "groups_allowed": [ - 1 - ], - "name": "raw noop", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 3 - }, - { - "fields": { - "description": "a method to complement strings of DNA", - "groups_allowed": [ - 1 - ], - "name": "DNA complement", - "user": 3, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 4 - }, - { - "fields": { - "description": "a method to reverse strings of DNA", - "groups_allowed": [ - 1 - ], - "name": "DNA reverse", - "user": 3, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 5 - }, - { - "fields": { - "description": "a method to turn strings of DNA into RNA", - "groups_allowed": [ - 1 - ], - "name": "DNA to RNA", - "user": 3, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 6 - }, - { - "fields": { - "description": "a pipeline to complement DNA", - "groups_allowed": [ - 1 - ], - "name": "DNA complement", - "user": 3, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 1 - }, - { - "fields": { - "description": "a pipeline to reverse DNA", - "groups_allowed": [ - 1 - ], - "name": "DNA reverse", - "user": 3, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 2 - }, - { - "fields": { - "description": "a pipeline to reverse and complement DNA", - "groups_allowed": [ - 1 - ], - "name": "DNA revcomp", - "user": 3, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 3 - }, - { - "fields": { - "description": "a pipeline to reverse DNA and translate it to RNA", - "groups_allowed": [ - 1 - ], - "name": "DNA to reversed RNA", - "user": 3, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 4 - }, - { - "fields": { - "family": 1, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 4 - }, - { - "fields": { - "family": 2, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 5 - }, - { - "fields": { - "family": 3, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 6 - }, - { - "fields": { - "family": 1, - "groups_allowed": [], - "published": false, - "revision_number": 2, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 9 - }, - { - "fields": { - "family": 3, - "groups_allowed": [], - "published": false, - "revision_number": 2, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 10 - }, - { - "fields": { - "family": 3, - "groups_allowed": [], - "published": false, - "revision_number": 3, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 11 - }, - { - "fields": { - "family": 4, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 13 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 1 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 2 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 3 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 3, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 4 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 3, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 3, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 6 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 3, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 7 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 3, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 8 - }, - { - "fields": { - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:01Z", - "revision_desc": "second version", - "revision_name": "2", - "user": 3, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 9 - }, - { - "fields": { - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "second version", - "revision_name": "2", - "user": 3, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 10 - }, - { - "fields": { - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "third version", - "revision_name": "3", - "user": 3, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 11 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:03Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 3, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 12 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:03Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 3, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 13 - }, - { - "fields": { - "dataset": 1, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 3 - }, - "model": "datachecking.contentchecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "end_time": "2000-01-01T00:00:17Z", - "execlog": 2, - "start_time": "2000-01-01T00:00:16Z", - "user": 3 - }, - "model": "datachecking.contentchecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 3, - "end_time": "2000-01-01T00:00:31Z", - "execlog": 4, - "start_time": "2000-01-01T00:00:31Z", - "user": 3 - }, - "model": "datachecking.contentchecklog", - "pk": 3 - }, - { - "fields": { - "dataset": 1, - "end_time": "2000-01-01T00:00:05Z", - "execlog": null, - "read_failed": false, - "runcomponent": 3, - "start_time": "2000-01-01T00:00:05Z", - "user": 3 - }, - "model": "datachecking.integritychecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "end_time": "2000-01-01T00:00:16Z", - "execlog": 2, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:16Z", - "user": 3 - }, - "model": "datachecking.integritychecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 3, - "end_time": "2000-01-01T00:00:31Z", - "execlog": 4, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:31Z", - "user": 3 - }, - "model": "datachecking.integritychecklog", - "pk": 3 - } -] \ No newline at end of file diff --git a/kive/portal/fixtures/execute_result_tests_rm.json b/kive/portal/fixtures/execute_result_tests_rm.json deleted file mode 100644 index df6c2a937..000000000 --- a/kive/portal/fixtures/execute_result_tests_rm.json +++ /dev/null @@ -1,2770 +0,0 @@ -[ - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "word", - "compounddatatype": 5, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 7 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "header", - "compounddatatype": 6, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 8 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "sequence", - "compounddatatype": 6, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 9 - }, - { - "fields": { - "dataset": 1, - "index": 1, - "run": 1 - }, - "model": "archive.runinput", - "pk": 1 - }, - { - "fields": { - "dataset": 1, - "index": 1, - "run": 2 - }, - "model": "archive.runinput", - "pk": 2 - }, - { - "fields": { - "dataset": 1, - "index": 1, - "run": 3 - }, - "model": "archive.runinput", - "pk": 3 - }, - { - "fields": { - "dataset": 1, - "index": 1, - "run": 4 - }, - "model": "archive.runinput", - "pk": 4 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:16Z", - "execrecord": 2, - "reused": false, - "start_time": "2000-01-01T00:00:04Z" - }, - "model": "archive.runcomponent", - "pk": 1 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:05Z", - "execrecord": 1, - "reused": false, - "start_time": "2000-01-01T00:00:04Z" - }, - "model": "archive.runcomponent", - "pk": 2 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:18Z", - "execrecord": 3, - "reused": false, - "start_time": "2000-01-01T00:00:18Z" - }, - "model": "archive.runcomponent", - "pk": 3 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:20Z", - "execrecord": 2, - "reused": true, - "start_time": "2000-01-01T00:00:20Z" - }, - "model": "archive.runcomponent", - "pk": 4 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:20Z", - "execrecord": 1, - "reused": true, - "start_time": "2000-01-01T00:00:20Z" - }, - "model": "archive.runcomponent", - "pk": 5 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:20Z", - "execrecord": 3, - "reused": true, - "start_time": "2000-01-01T00:00:20Z" - }, - "model": "archive.runcomponent", - "pk": 6 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:34Z", - "execrecord": 4, - "reused": false, - "start_time": "2000-01-01T00:00:21Z" - }, - "model": "archive.runcomponent", - "pk": 7 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:21Z", - "execrecord": 1, - "reused": false, - "start_time": "2000-01-01T00:00:21Z" - }, - "model": "archive.runcomponent", - "pk": 8 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:36Z", - "execrecord": 5, - "reused": false, - "start_time": "2000-01-01T00:00:36Z" - }, - "model": "archive.runcomponent", - "pk": 9 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:38Z", - "execrecord": 4, - "reused": true, - "start_time": "2000-01-01T00:00:38Z" - }, - "model": "archive.runcomponent", - "pk": 10 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:52Z", - "execrecord": 6, - "reused": false, - "start_time": "2000-01-01T00:00:38Z" - }, - "model": "archive.runcomponent", - "pk": 11 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:38Z", - "execrecord": 1, - "reused": true, - "start_time": "2000-01-01T00:00:38Z" - }, - "model": "archive.runcomponent", - "pk": 12 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:39Z", - "execrecord": 5, - "reused": false, - "start_time": "2000-01-01T00:00:38Z" - }, - "model": "archive.runcomponent", - "pk": 13 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:54Z", - "execrecord": 7, - "reused": false, - "start_time": "2000-01-01T00:00:54Z" - }, - "model": "archive.runcomponent", - "pk": 14 - }, - { - "fields": { - "pipelinestep": 1, - "run": 1 - }, - "model": "archive.runstep", - "pk": 1 - }, - { - "fields": { - "pipelinestep": 1, - "run": 2 - }, - "model": "archive.runstep", - "pk": 4 - }, - { - "fields": { - "pipelinestep": 2, - "run": 3 - }, - "model": "archive.runstep", - "pk": 7 - }, - { - "fields": { - "pipelinestep": 3, - "run": 4 - }, - "model": "archive.runstep", - "pk": 10 - }, - { - "fields": { - "pipelinestep": 4, - "run": 4 - }, - "model": "archive.runstep", - "pk": 11 - }, - { - "fields": { - "PSIC": 1, - "dest_runstep": 1 - }, - "model": "archive.runsic", - "pk": 2 - }, - { - "fields": { - "PSIC": 1, - "dest_runstep": 4 - }, - "model": "archive.runsic", - "pk": 5 - }, - { - "fields": { - "PSIC": 3, - "dest_runstep": 7 - }, - "model": "archive.runsic", - "pk": 8 - }, - { - "fields": { - "PSIC": 5, - "dest_runstep": 10 - }, - "model": "archive.runsic", - "pk": 12 - }, - { - "fields": { - "PSIC": 6, - "dest_runstep": 11 - }, - "model": "archive.runsic", - "pk": 13 - }, - { - "fields": { - "pipelineoutputcable": 2, - "run": 1 - }, - "model": "archive.runoutputcable", - "pk": 3 - }, - { - "fields": { - "pipelineoutputcable": 2, - "run": 2 - }, - "model": "archive.runoutputcable", - "pk": 6 - }, - { - "fields": { - "pipelineoutputcable": 4, - "run": 3 - }, - "model": "archive.runoutputcable", - "pk": 9 - }, - { - "fields": { - "pipelineoutputcable": 7, - "run": 4 - }, - "model": "archive.runoutputcable", - "pk": 14 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:04Z", - "invoking_record": 2, - "record": 2, - "start_time": "2000-01-01T00:00:04Z" - }, - "model": "archive.execlog", - "pk": 1 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:05Z", - "invoking_record": 1, - "record": 1, - "start_time": "2000-01-01T00:00:05Z" - }, - "model": "archive.execlog", - "pk": 2 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:18Z", - "invoking_record": 3, - "record": 3, - "start_time": "2000-01-01T00:00:18Z" - }, - "model": "archive.execlog", - "pk": 3 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:21Z", - "invoking_record": 8, - "record": 8, - "start_time": "2000-01-01T00:00:21Z" - }, - "model": "archive.execlog", - "pk": 4 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:21Z", - "invoking_record": 7, - "record": 7, - "start_time": "2000-01-01T00:00:21Z" - }, - "model": "archive.execlog", - "pk": 5 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:36Z", - "invoking_record": 9, - "record": 9, - "start_time": "2000-01-01T00:00:36Z" - }, - "model": "archive.execlog", - "pk": 6 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:38Z", - "invoking_record": 13, - "record": 13, - "start_time": "2000-01-01T00:00:38Z" - }, - "model": "archive.execlog", - "pk": 7 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:39Z", - "invoking_record": 11, - "record": 11, - "start_time": "2000-01-01T00:00:39Z" - }, - "model": "archive.execlog", - "pk": 8 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:54Z", - "invoking_record": 14, - "record": 14, - "start_time": "2000-01-01T00:00:54Z" - }, - "model": "archive.execlog", - "pk": 9 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step1_stderr_slurmIDJ_nodeN.txt", - "error_redacted": false, - "execlog": 2, - "install_failed": false, - "output_log": "Logs/step1_stdout_slurmIDJ_nodeN.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 1 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step1_stderr_slurmIDJ_nodeN_PzqGpVz.txt", - "error_redacted": false, - "execlog": 5, - "install_failed": false, - "output_log": "Logs/step1_stdout_slurmIDJ_nodeN_GDD57nW.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 2 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step2_stderr_slurmIDJ_nodeN.txt", - "error_redacted": false, - "execlog": 8, - "install_failed": false, - "output_log": "Logs/step2_stdout_slurmIDJ_nodeN.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 6, - "dataset": 1, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 6, - "dataset": 2, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 6, - "dataset": 3, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 6, - "dataset": 4, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 4 - }, - { - "fields": { - "generator": 1 - }, - "model": "librarian.execrecord", - "pk": 1 - }, - { - "fields": { - "generator": 2 - }, - "model": "librarian.execrecord", - "pk": 2 - }, - { - "fields": { - "generator": 3 - }, - "model": "librarian.execrecord", - "pk": 3 - }, - { - "fields": { - "generator": 5 - }, - "model": "librarian.execrecord", - "pk": 4 - }, - { - "fields": { - "generator": 6 - }, - "model": "librarian.execrecord", - "pk": 5 - }, - { - "fields": { - "generator": 8 - }, - "model": "librarian.execrecord", - "pk": 6 - }, - { - "fields": { - "generator": 9 - }, - "model": "librarian.execrecord", - "pk": 7 - }, - { - "fields": { - "dataset": 1, - "execrecord": 1, - "generic_input": 11 - }, - "model": "librarian.execrecordin", - "pk": 1 - }, - { - "fields": { - "dataset": 1, - "execrecord": 2, - "generic_input": 7 - }, - "model": "librarian.execrecordin", - "pk": 2 - }, - { - "fields": { - "dataset": 2, - "execrecord": 3, - "generic_input": 8 - }, - "model": "librarian.execrecordin", - "pk": 3 - }, - { - "fields": { - "dataset": 1, - "execrecord": 4, - "generic_input": 9 - }, - "model": "librarian.execrecordin", - "pk": 4 - }, - { - "fields": { - "dataset": 3, - "execrecord": 5, - "generic_input": 10 - }, - "model": "librarian.execrecordin", - "pk": 5 - }, - { - "fields": { - "dataset": 3, - "execrecord": 6, - "generic_input": 7 - }, - "model": "librarian.execrecordin", - "pk": 6 - }, - { - "fields": { - "dataset": 4, - "execrecord": 7, - "generic_input": 8 - }, - "model": "librarian.execrecordin", - "pk": 7 - }, - { - "fields": { - "dataset": 1, - "execrecord": 1, - "generic_output": 7 - }, - "model": "librarian.execrecordout", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "execrecord": 2, - "generic_output": 8 - }, - "model": "librarian.execrecordout", - "pk": 2 - }, - { - "fields": { - "dataset": 2, - "execrecord": 3, - "generic_output": 13 - }, - "model": "librarian.execrecordout", - "pk": 3 - }, - { - "fields": { - "dataset": 3, - "execrecord": 4, - "generic_output": 10 - }, - "model": "librarian.execrecordout", - "pk": 4 - }, - { - "fields": { - "dataset": 3, - "execrecord": 5, - "generic_output": 16 - }, - "model": "librarian.execrecordout", - "pk": 5 - }, - { - "fields": { - "dataset": 4, - "execrecord": 6, - "generic_output": 8 - }, - "model": "librarian.execrecordout", - "pk": 6 - }, - { - "fields": { - "dataset": 4, - "execrecord": 7, - "generic_output": 19 - }, - "model": "librarian.execrecordout", - "pk": 7 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 4, - "step_num": 1, - "transformation": 7, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 1 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 1, - "transformation": 8, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 2 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 6, - "step_num": 1, - "transformation": 8, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 3 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 6, - "step_num": 2, - "transformation": 7, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 4 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [ - 8 - ], - "pipeline": 9, - "step_num": 1, - "transformation": 7, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 5 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [ - 10 - ], - "pipeline": 10, - "step_num": 1, - "transformation": 8, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 6 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [ - 8 - ], - "pipeline": 10, - "step_num": 2, - "transformation": 7, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 7 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [ - 10 - ], - "pipeline": 11, - "step_num": 1, - "transformation": 8, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 8 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 11, - "step_num": 2, - "transformation": 7, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 9 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 13, - "step_num": 1, - "transformation": 8, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 10 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 13, - "step_num": 2, - "transformation": 12, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 11 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 1 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 2 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 3 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 4 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 5 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 6 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 7 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 8 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 10 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 11 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 13 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 14 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 15 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 16 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 17 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 18 - }, - { - "fields": { - "dest": 7, - "keep_output": false, - "pipelinestep": 1, - "source": 11, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 1 - }, - { - "fields": { - "dest": 9, - "keep_output": false, - "pipelinestep": 2, - "source": 14, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 3 - }, - { - "fields": { - "dest": 9, - "keep_output": false, - "pipelinestep": 3, - "source": 17, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 5 - }, - { - "fields": { - "dest": 7, - "keep_output": false, - "pipelinestep": 4, - "source": 10, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 6 - }, - { - "fields": { - "dest": 7, - "keep_output": false, - "pipelinestep": 5, - "source": 20, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 8 - }, - { - "fields": { - "dest": 9, - "keep_output": false, - "pipelinestep": 6, - "source": 22, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 10 - }, - { - "fields": { - "dest": 7, - "keep_output": false, - "pipelinestep": 7, - "source": 10, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 11 - }, - { - "fields": { - "dest": 9, - "keep_output": false, - "pipelinestep": 8, - "source": 24, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 13 - }, - { - "fields": { - "dest": 7, - "keep_output": false, - "pipelinestep": 9, - "source": 10, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 14 - }, - { - "fields": { - "dest": 9, - "keep_output": false, - "pipelinestep": 10, - "source": 29, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 16 - }, - { - "fields": { - "dest": 27, - "keep_output": false, - "pipelinestep": 11, - "source": 10, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 17 - }, - { - "fields": { - "output_cdt": 6, - "output_idx": 1, - "output_name": "complemented_lab_data", - "pipeline": 4, - "source": 8, - "source_step": 1 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 2 - }, - { - "fields": { - "output_cdt": 6, - "output_idx": 1, - "output_name": "reversed_lab_data", - "pipeline": 5, - "source": 10, - "source_step": 1 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 4 - }, - { - "fields": { - "output_cdt": 6, - "output_idx": 1, - "output_name": "reverse_and_complemented_lab_data", - "pipeline": 6, - "source": 8, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 7 - }, - { - "fields": { - "output_cdt": 6, - "output_idx": 1, - "output_name": "revcomped_lab_data", - "pipeline": 11, - "source": 8, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 15 - }, - { - "fields": { - "output_cdt": 6, - "output_idx": 1, - "output_name": "RNAd_lab_data", - "pipeline": 13, - "source": 28, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 18 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 1 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 2 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 3 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 4 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 5 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 6 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 7 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 8 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 9 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 10 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 11 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 13 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 14 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 16 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 17 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 19 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 20 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 22 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 24 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 26 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 27 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 28 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 29 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 31 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 1 - }, - "model": "transformation.xputstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 2 - }, - "model": "transformation.xputstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 3 - }, - "model": "transformation.xputstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 4 - }, - "model": "transformation.xputstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 7 - }, - "model": "transformation.xputstructure", - "pk": 5 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 8 - }, - "model": "transformation.xputstructure", - "pk": 6 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 9 - }, - "model": "transformation.xputstructure", - "pk": 7 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 10 - }, - "model": "transformation.xputstructure", - "pk": 8 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 11 - }, - "model": "transformation.xputstructure", - "pk": 9 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 13 - }, - "model": "transformation.xputstructure", - "pk": 11 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 14 - }, - "model": "transformation.xputstructure", - "pk": 12 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 16 - }, - "model": "transformation.xputstructure", - "pk": 14 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 17 - }, - "model": "transformation.xputstructure", - "pk": 15 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 19 - }, - "model": "transformation.xputstructure", - "pk": 17 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 20 - }, - "model": "transformation.xputstructure", - "pk": 18 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 22 - }, - "model": "transformation.xputstructure", - "pk": 20 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 24 - }, - "model": "transformation.xputstructure", - "pk": 22 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 26 - }, - "model": "transformation.xputstructure", - "pk": 24 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 27 - }, - "model": "transformation.xputstructure", - "pk": 25 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 28 - }, - "model": "transformation.xputstructure", - "pk": 26 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 29 - }, - "model": "transformation.xputstructure", - "pk": 27 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 31 - }, - "model": "transformation.xputstructure", - "pk": 29 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "strings", - "transformation": 1 - }, - "model": "transformation.transformationinput", - "pk": 1 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "strings", - "transformation": 2 - }, - "model": "transformation.transformationinput", - "pk": 3 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "raw", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 5 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "DNA_to_complement", - "transformation": 7 - }, - "model": "transformation.transformationinput", - "pk": 7 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "DNA_to_reverse", - "transformation": 8 - }, - "model": "transformation.transformationinput", - "pk": 9 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "lab_data", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 11 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "lab_data", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 14 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "lab_data", - "transformation": 6 - }, - "model": "transformation.transformationinput", - "pk": 17 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "lab_data", - "transformation": 9 - }, - "model": "transformation.transformationinput", - "pk": 20 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "lab_data", - "transformation": 10 - }, - "model": "transformation.transformationinput", - "pk": 22 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "lab_data", - "transformation": 11 - }, - "model": "transformation.transformationinput", - "pk": 24 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "DNA_to_convert", - "transformation": 12 - }, - "model": "transformation.transformationinput", - "pk": 27 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "lab_data", - "transformation": 13 - }, - "model": "transformation.transformationinput", - "pk": 29 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "same_strings", - "transformation": 1 - }, - "model": "transformation.transformationoutput", - "pk": 2 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "untouched_strings", - "transformation": 2 - }, - "model": "transformation.transformationoutput", - "pk": 4 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "same_raw", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 6 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "complemented_DNA", - "transformation": 7 - }, - "model": "transformation.transformationoutput", - "pk": 8 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "reversed_DNA", - "transformation": 8 - }, - "model": "transformation.transformationoutput", - "pk": 10 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "complemented_lab_data", - "transformation": 4 - }, - "model": "transformation.transformationoutput", - "pk": 13 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "reversed_lab_data", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 16 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "reverse_and_complemented_lab_data", - "transformation": 6 - }, - "model": "transformation.transformationoutput", - "pk": 19 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "revcomped_lab_data", - "transformation": 11 - }, - "model": "transformation.transformationoutput", - "pk": 26 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "RNA", - "transformation": 12 - }, - "model": "transformation.transformationoutput", - "pk": 28 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "RNAd_lab_data", - "transformation": 13 - }, - "model": "transformation.transformationoutput", - "pk": 31 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:00Z", - "email": "bob@talabs.com", - "first_name": "", - "groups": [ - 1 - ], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$7YkoRdCFf8jj$Gt4ZGnKWTUZ+/ptpx+XdGhvk2PkRLiAyC0B6llKf1SQ=", - "user_permissions": [], - "username": "bob" - }, - "model": "auth.user", - "pk": 2 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:00Z", - "email": "alice@talabs.com", - "first_name": "", - "groups": [ - 1 - ], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$xeeHZgrO7yrq$sLp0WVB4XL7NQTX+9IwWOqhzhr7I/1m4R3E9VDMtmvI=", - "user_permissions": [], - "username": "alice" - }, - "model": "auth.user", - "pk": 3 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:00Z", - "description": "sequences of ASCII characters", - "groups_allowed": [ - 1 - ], - "name": "my_string", - "prototype": null, - "restricts": [ - 1 - ], - "user": 2, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 8 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:00Z", - "description": "sequences of ATCG", - "groups_allowed": [ - 1 - ], - "name": "DNA", - "prototype": null, - "restricts": [ - 1 - ], - "user": 3, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 9 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 3, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 6 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:20Z", - "groups_allowed": [], - "name": "", - "parent_runstep": null, - "paused_by": null, - "pipeline": 4, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "/home/rliang/Documents/Kive/media/Sandboxes/useralice_run1_aMgneh", - "start_time": "2000-01-01T00:00:04Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:57:53.616Z", - "user": 3, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 1 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:20Z", - "groups_allowed": [], - "name": "", - "parent_runstep": null, - "paused_by": null, - "pipeline": 4, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "/home/rliang/Documents/Kive/media/Sandboxes/useralice_run2_NC3a7e", - "start_time": "2000-01-01T00:00:20Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:58:09.293Z", - "user": 3, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 2 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:37Z", - "groups_allowed": [], - "name": "", - "parent_runstep": null, - "paused_by": null, - "pipeline": 5, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "/home/rliang/Documents/Kive/media/Sandboxes/useralice_run3_WRq0eV", - "start_time": "2000-01-01T00:00:21Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:58:10.087Z", - "user": 3, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 3 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:55Z", - "groups_allowed": [], - "name": "", - "parent_runstep": null, - "paused_by": null, - "pipeline": 6, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "/home/rliang/Documents/Kive/media/Sandboxes/useralice_run4_F0DdJj", - "start_time": "2000-01-01T00:00:37Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:58:26.954Z", - "user": 3, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 4 - }, - { - "fields": { - "MD5_checksum": "0082bc75b97d81a1b92adaa894fef1bd", - "_redacted": false, - "dataset_file": "Datasets/2017_04/tmp3KEWCd", - "date_created": "2000-01-01T00:00:01Z", - "description": "data from the lab", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [], - "last_time_checked": "2017-04-25T00:57:50.935Z", - "name": "lab data", - "user": 3, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "c0d4cb49470a30a88ea5d4cf0c3324c9", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step1_complemented_DNA.csv", - "date_created": "2000-01-01T00:00:15Z", - "description": "Generated data from a run of pipeline \"DNA complement:1 (v1)\" started at 2017-04-25 00:57:53.722472+00:00 by alice\nrun: 1\nuser: alice\nstep: 1\noutput: complemented_DNA", - "external_path": "", - "externalfiledirectory": null, - "file_source": 1, - "groups_allowed": [], - "last_time_checked": "2017-04-25T00:58:04.957Z", - "name": "run1_step1_outputcomplemented_DNA", - "user": 3, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "c0d4cb49470a30a88ea5d4cf0c3324c9", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step1_reversed_DNA.csv", - "date_created": "2000-01-01T00:00:33Z", - "description": "Generated data from a run of pipeline \"DNA reverse:1 (v1)\" started at 2017-04-25 00:58:10.215870+00:00 by alice\nrun: 3\nuser: alice\nstep: 1\noutput: reversed_DNA", - "external_path": "", - "externalfiledirectory": null, - "file_source": 7, - "groups_allowed": [], - "last_time_checked": "2017-04-25T00:58:22.612Z", - "name": "run3_step1_outputreversed_DNA", - "user": 3, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 3 - }, - { - "fields": { - "MD5_checksum": "0082bc75b97d81a1b92adaa894fef1bd", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step2_complemented_DNA.csv", - "date_created": "2000-01-01T00:00:51Z", - "description": "Generated data from a run of pipeline \"DNA revcomp:1 (v1)\" started at 2017-04-25 00:58:27.096237+00:00 by alice\nrun: 4\nuser: alice\nstep: 2\noutput: complemented_DNA", - "external_path": "", - "externalfiledirectory": null, - "file_source": 11, - "groups_allowed": [], - "last_time_checked": "2017-04-25T00:58:40.858Z", - "name": "run4_step2_outputcomplemented_DNA", - "user": 3, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 4 - }, - { - "fields": { - "description": "a script to do nothing", - "filename": "noop.sh", - "groups_allowed": [ - 1 - ], - "name": "noop", - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 1 - }, - { - "fields": { - "description": "a script to complement DNA", - "filename": "complement.sh", - "groups_allowed": [ - 1 - ], - "name": "DNA complement", - "user": 3, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 2 - }, - { - "fields": { - "description": "a script to reverse DNA", - "filename": "reverse.sh", - "groups_allowed": [ - 1 - ], - "name": "DNA reverse", - "user": 3, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 3 - }, - { - "fields": { - "description": "a script to reverse DNA", - "filename": "DNA2RNA.sh", - "groups_allowed": [ - 1 - ], - "name": "DNA to RNA", - "user": 3, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 4 - }, - { - "fields": { - "MD5_checksum": "d218c00725d473639f347d456de381d8", - "coderesource": 1, - "content_file": "CodeResources/fdopen", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "1", - "revision_number": 1, - "revision_parent": null, - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "5699120b018a7932c07be0412483c622", - "coderesource": 2, - "content_file": "CodeResources/fdopen_gAd1pjF", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "1", - "revision_number": 1, - "revision_parent": null, - "user": 3, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "196144423c5bb98d59d78c729ce842f4", - "coderesource": 3, - "content_file": "CodeResources/fdopen_PCeMOY7", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "1", - "revision_number": 1, - "revision_parent": null, - "user": 3, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 3 - }, - { - "fields": { - "MD5_checksum": "30e9a1ac1dad0c134d41bf826a53ba68", - "coderesource": 4, - "content_file": "CodeResources/fdopen_5kq9pyX", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "1", - "revision_number": 1, - "revision_parent": null, - "user": 3, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 4 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 1 - }, - { - "fields": { - "driver": 1, - "family": 2, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 2 - }, - { - "fields": { - "driver": 1, - "family": 3, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 3 - }, - { - "fields": { - "driver": 2, - "family": 4, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 7 - }, - { - "fields": { - "driver": 2, - "family": 5, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 8 - }, - { - "fields": { - "driver": 4, - "family": 6, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 12 - }, - { - "fields": { - "description": "a method to do nothing to strings", - "groups_allowed": [ - 1 - ], - "name": "string noop", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 1 - }, - { - "fields": { - "description": "a TOTALLY DIFFERENT method that TOTALLY does SOMETHING to strings by leaving them alone", - "groups_allowed": [ - 1 - ], - "name": "string trivial", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 2 - }, - { - "fields": { - "description": "do nothing to raw data", - "groups_allowed": [ - 1 - ], - "name": "raw noop", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 3 - }, - { - "fields": { - "description": "a method to complement strings of DNA", - "groups_allowed": [ - 1 - ], - "name": "DNA complement", - "user": 3, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 4 - }, - { - "fields": { - "description": "a method to reverse strings of DNA", - "groups_allowed": [ - 1 - ], - "name": "DNA reverse", - "user": 3, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 5 - }, - { - "fields": { - "description": "a method to turn strings of DNA into RNA", - "groups_allowed": [ - 1 - ], - "name": "DNA to RNA", - "user": 3, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 6 - }, - { - "fields": { - "description": "a pipeline to complement DNA", - "groups_allowed": [ - 1 - ], - "name": "DNA complement", - "user": 3, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 1 - }, - { - "fields": { - "description": "a pipeline to reverse DNA", - "groups_allowed": [ - 1 - ], - "name": "DNA reverse", - "user": 3, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 2 - }, - { - "fields": { - "description": "a pipeline to reverse and complement DNA", - "groups_allowed": [ - 1 - ], - "name": "DNA revcomp", - "user": 3, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 3 - }, - { - "fields": { - "description": "a pipeline to reverse DNA and translate it to RNA", - "groups_allowed": [ - 1 - ], - "name": "DNA to reversed RNA", - "user": 3, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 4 - }, - { - "fields": { - "family": 1, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 4 - }, - { - "fields": { - "family": 2, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 5 - }, - { - "fields": { - "family": 3, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 6 - }, - { - "fields": { - "family": 1, - "groups_allowed": [], - "published": false, - "revision_number": 2, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 9 - }, - { - "fields": { - "family": 3, - "groups_allowed": [], - "published": false, - "revision_number": 2, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 10 - }, - { - "fields": { - "family": 3, - "groups_allowed": [], - "published": false, - "revision_number": 3, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 11 - }, - { - "fields": { - "family": 4, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 13 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 1 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 2 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 3 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 3, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 4 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 3, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 3, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 6 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 3, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 7 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 3, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 8 - }, - { - "fields": { - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:01Z", - "revision_desc": "second version", - "revision_name": "2", - "user": 3, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 9 - }, - { - "fields": { - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "second version", - "revision_name": "2", - "user": 3, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 10 - }, - { - "fields": { - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "third version", - "revision_name": "3", - "user": 3, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 11 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 3, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 12 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:03Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 3, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 13 - }, - { - "fields": { - "dataset": 1, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 3 - }, - "model": "datachecking.contentchecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "end_time": "2000-01-01T00:00:16Z", - "execlog": 2, - "start_time": "2000-01-01T00:00:16Z", - "user": 3 - }, - "model": "datachecking.contentchecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 3, - "end_time": "2000-01-01T00:00:34Z", - "execlog": 5, - "start_time": "2000-01-01T00:00:34Z", - "user": 3 - }, - "model": "datachecking.contentchecklog", - "pk": 3 - }, - { - "fields": { - "dataset": 4, - "end_time": "2000-01-01T00:00:52Z", - "execlog": 8, - "start_time": "2000-01-01T00:00:52Z", - "user": 3 - }, - "model": "datachecking.contentchecklog", - "pk": 4 - }, - { - "fields": { - "dataset": 1, - "end_time": "2000-01-01T00:00:04Z", - "execlog": null, - "read_failed": false, - "runcomponent": 2, - "start_time": "2000-01-01T00:00:04Z", - "user": 3 - }, - "model": "datachecking.integritychecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "end_time": "2000-01-01T00:00:16Z", - "execlog": 2, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:16Z", - "user": 3 - }, - "model": "datachecking.integritychecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 1, - "end_time": "2000-01-01T00:00:21Z", - "execlog": null, - "read_failed": false, - "runcomponent": 8, - "start_time": "2000-01-01T00:00:21Z", - "user": 3 - }, - "model": "datachecking.integritychecklog", - "pk": 3 - }, - { - "fields": { - "dataset": 3, - "end_time": "2000-01-01T00:00:34Z", - "execlog": 5, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:34Z", - "user": 3 - }, - "model": "datachecking.integritychecklog", - "pk": 4 - }, - { - "fields": { - "dataset": 3, - "end_time": "2000-01-01T00:00:38Z", - "execlog": null, - "read_failed": false, - "runcomponent": 13, - "start_time": "2000-01-01T00:00:38Z", - "user": 3 - }, - "model": "datachecking.integritychecklog", - "pk": 5 - }, - { - "fields": { - "dataset": 4, - "end_time": "2000-01-01T00:00:52Z", - "execlog": 8, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:52Z", - "user": 3 - }, - "model": "datachecking.integritychecklog", - "pk": 6 - } -] \ No newline at end of file diff --git a/kive/portal/fixtures/execute_tests.json b/kive/portal/fixtures/execute_tests.json deleted file mode 100644 index 158f7d67e..000000000 --- a/kive/portal/fixtures/execute_tests.json +++ /dev/null @@ -1,648 +0,0 @@ -[ - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "pX_a", - "compounddatatype": 5, - "datatype": 4 - }, - "model": "metadata.compounddatatypemember", - "pk": 7 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "pX_b", - "compounddatatype": 5, - "datatype": 4 - }, - "model": "metadata.compounddatatypemember", - "pk": 8 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "pX_c", - "compounddatatype": 5, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 9 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "a", - "compounddatatype": 6, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 10 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "b", - "compounddatatype": 6, - "datatype": 4 - }, - "model": "metadata.compounddatatypemember", - "pk": 11 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "c", - "compounddatatype": 7, - "datatype": 4 - }, - "model": "metadata.compounddatatypemember", - "pk": 12 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "d", - "compounddatatype": 7, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 13 - }, - { - "fields": { - "compounddatatype": 5, - "dataset": 1, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 1 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 2, - "step_num": 1, - "transformation": 1, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 1 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 3, - "step_num": 1, - "transformation": 4, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 2 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 1 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 2 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 3 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 4 - }, - { - "fields": { - "dest": 1, - "keep_output": false, - "pipelinestep": 1, - "source": 3, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 1 - }, - { - "fields": { - "dest": 5, - "keep_output": false, - "pipelinestep": 2, - "source": 7, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 3 - }, - { - "fields": { - "cable": 1, - "dest_pin": 11, - "source_pin": 8 - }, - "model": "pipeline.customcablewire", - "pk": 1 - }, - { - "fields": { - "cable": 1, - "dest_pin": 10, - "source_pin": 9 - }, - "model": "pipeline.customcablewire", - "pk": 2 - }, - { - "fields": { - "output_cdt": 7, - "output_idx": 1, - "output_name": "pX_out", - "pipeline": 2, - "source": 2, - "source_step": 1 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 2 - }, - { - "fields": { - "output_cdt": 7, - "output_idx": 1, - "output_name": "pX_out", - "pipeline": 3, - "source": 6, - "source_step": 1 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 4 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 1 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 2 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 3 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 4 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 5 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 6 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 7 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 8 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 1 - }, - "model": "transformation.xputstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 7, - "max_row": null, - "min_row": null, - "transf_xput": 2 - }, - "model": "transformation.xputstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 3 - }, - "model": "transformation.xputstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 7, - "max_row": null, - "min_row": null, - "transf_xput": 4 - }, - "model": "transformation.xputstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 7, - "max_row": null, - "min_row": null, - "transf_xput": 6 - }, - "model": "transformation.xputstructure", - "pk": 5 - }, - { - "fields": { - "compounddatatype": 7, - "max_row": null, - "min_row": null, - "transf_xput": 8 - }, - "model": "transformation.xputstructure", - "pk": 6 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "mA_in", - "transformation": 1 - }, - "model": "transformation.transformationinput", - "pk": 1 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "pX_in", - "transformation": 2 - }, - "model": "transformation.transformationinput", - "pk": 3 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "mA_in", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 5 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "pX_in", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 7 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "mA_out", - "transformation": 1 - }, - "model": "transformation.transformationoutput", - "pk": 2 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "pX_out", - "transformation": 2 - }, - "model": "transformation.transformationoutput", - "pk": 4 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "mA_out", - "transformation": 4 - }, - "model": "transformation.transformationoutput", - "pk": 6 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "pX_out", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 8 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:00Z", - "email": "lennon@thebeatles.com", - "first_name": "", - "groups": [ - 1 - ], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$2yEKsuIbWNTg$1una8+sxu0pum+r65U5CnrCLLd5kP6U2Vax5i/M4gSU=", - "user_permissions": [], - "username": "john" - }, - "model": "auth.user", - "pk": 2 - }, - { - "fields": { - "groups_allowed": [], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 6 - }, - { - "fields": { - "groups_allowed": [], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 7 - }, - { - "fields": { - "MD5_checksum": "4ce50c714420255e82dd3965ccec07ec", - "_redacted": false, - "dataset_file": "Datasets/2017_04/input_for_test_C_twostep_with_subpipeline.csv", - "date_created": "2000-01-01T00:00:00Z", - "description": "input to pipeline pX", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [], - "last_time_checked": "2017-04-25T01:00:00.600Z", - "name": "pX_in_dataset", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "4ce50c714420255e82dd3965ccec07ec", - "_redacted": false, - "dataset_file": "Datasets/2017_04/input_for_test_C_twostep_with_subpipeline_oot3bnr.csv", - "date_created": "2000-01-01T00:00:00Z", - "description": "input to pipeline pX", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [], - "last_time_checked": "2017-04-25T01:00:00.679Z", - "name": "pX_in_dataset", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 2 - }, - { - "fields": { - "description": "self.mA_cr desc", - "filename": "mA.py", - "groups_allowed": [], - "name": "mA_CR", - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "e353a62fe8cb6a56d31b8c65d70b217d", - "coderesource": 1, - "content_file": "CodeResources/generic_script.py", - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "desc", - "revision_name": "v1", - "revision_number": 1, - "revision_parent": null, - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 1 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 1 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [], - "reusable": 1, - "revision_number": 2, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 4 - }, - { - "fields": { - "description": "self.mf desc", - "groups_allowed": [], - "name": "self.mf", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 1 - }, - { - "fields": { - "description": "self.pf desc", - "groups_allowed": [], - "name": "self.pf", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 1 - }, - { - "fields": { - "family": 1, - "groups_allowed": [], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 2 - }, - { - "fields": { - "family": 1, - "groups_allowed": [], - "published": false, - "revision_number": 2, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 3 - }, - { - "fields": { - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "mA_desc", - "revision_name": "mA", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 1 - }, - { - "fields": { - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "X", - "revision_name": "pX_revision", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 2 - }, - { - "fields": { - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "X", - "revision_name": "pX_raw", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 3 - }, - { - "fields": { - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "mA_desc", - "revision_name": "mA_raw", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 4 - }, - { - "fields": { - "dataset": 1, - "end_time": "2000-01-01T00:00:00Z", - "execlog": null, - "start_time": "2000-01-01T00:00:00Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 1 - } -] \ No newline at end of file diff --git a/kive/portal/fixtures/find_datasets.json b/kive/portal/fixtures/find_datasets.json deleted file mode 100644 index ee9fe4213..000000000 --- a/kive/portal/fixtures/find_datasets.json +++ /dev/null @@ -1,1556 +0,0 @@ -[ - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "word", - "compounddatatype": 5, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 7 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "word", - "compounddatatype": 6, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 8 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "drow", - "compounddatatype": 6, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 9 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "drow", - "compounddatatype": 7, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 10 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "word", - "compounddatatype": 7, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 11 - }, - { - "fields": { - "compounddatatype": 5, - "dataset": 1, - "num_rows": 1 - }, - "model": "librarian.datasetstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 6, - "dataset": 2, - "num_rows": 20 - }, - "model": "librarian.datasetstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 7, - "dataset": 3, - "num_rows": 20 - }, - "model": "librarian.datasetstructure", - "pk": 3 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 8, - "step_num": 1, - "transformation": 1, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 1 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 9, - "step_num": 1, - "transformation": 4, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 2 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 9, - "step_num": 2, - "transformation": 7, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 3 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 10, - "step_num": 1, - "transformation": 7, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 4 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 10, - "step_num": 2, - "transformation": 9, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 5 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 10, - "step_num": 3, - "transformation": 7, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 6 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 1 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 2 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 3 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 4 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 5 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 6 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 7 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 8 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 9 - }, - { - "fields": { - "dest": 1, - "keep_output": false, - "pipelinestep": 1, - "source": 15, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 1 - }, - { - "fields": { - "dest": 7, - "keep_output": false, - "pipelinestep": 2, - "source": 18, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 3 - }, - { - "fields": { - "dest": 13, - "keep_output": false, - "pipelinestep": 3, - "source": 8, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 4 - }, - { - "fields": { - "dest": 13, - "keep_output": false, - "pipelinestep": 4, - "source": 20, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 6 - }, - { - "fields": { - "dest": 18, - "keep_output": false, - "pipelinestep": 5, - "source": 14, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 7 - }, - { - "fields": { - "dest": 13, - "keep_output": false, - "pipelinestep": 6, - "source": 19, - "source_step": 2 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 8 - }, - { - "fields": { - "cable": 3, - "dest_pin": 9, - "source_pin": 10 - }, - "model": "pipeline.customcablewire", - "pk": 1 - }, - { - "fields": { - "cable": 3, - "dest_pin": 8, - "source_pin": 11 - }, - "model": "pipeline.customcablewire", - "pk": 2 - }, - { - "fields": { - "cable": 4, - "dest_pin": 11, - "source_pin": 8 - }, - "model": "pipeline.customcablewire", - "pk": 3 - }, - { - "fields": { - "cable": 4, - "dest_pin": 10, - "source_pin": 9 - }, - "model": "pipeline.customcablewire", - "pk": 4 - }, - { - "fields": { - "cable": 8, - "dest_pin": 11, - "source_pin": 10 - }, - "model": "pipeline.customcablewire", - "pk": 5 - }, - { - "fields": { - "cable": 8, - "dest_pin": 10, - "source_pin": 11 - }, - "model": "pipeline.customcablewire", - "pk": 6 - }, - { - "fields": { - "output_cdt": 5, - "output_idx": 1, - "output_name": "complemented_lab_data", - "pipeline": 8, - "source": 2, - "source_step": 1 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 2 - }, - { - "fields": { - "output_cdt": 7, - "output_idx": 1, - "output_name": "reversed_words", - "pipeline": 9, - "source": 14, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 5 - }, - { - "fields": { - "output_cdt": 7, - "output_idx": 1, - "output_name": "unchanged_data", - "pipeline": 10, - "source": 14, - "source_step": 3 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 9 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 1 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 2 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 3 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 4 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 5 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 6 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 7 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 8 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 9 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 10 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 11 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 12 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 13 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 14 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 15 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 17 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 18 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 19 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 20 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 22 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 1 - }, - "model": "transformation.xputstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 2 - }, - "model": "transformation.xputstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 3 - }, - "model": "transformation.xputstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 4 - }, - "model": "transformation.xputstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 7 - }, - "model": "transformation.xputstructure", - "pk": 5 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 8 - }, - "model": "transformation.xputstructure", - "pk": 6 - }, - { - "fields": { - "compounddatatype": 7, - "max_row": null, - "min_row": null, - "transf_xput": 9 - }, - "model": "transformation.xputstructure", - "pk": 7 - }, - { - "fields": { - "compounddatatype": 7, - "max_row": null, - "min_row": null, - "transf_xput": 10 - }, - "model": "transformation.xputstructure", - "pk": 8 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 11 - }, - "model": "transformation.xputstructure", - "pk": 9 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 12 - }, - "model": "transformation.xputstructure", - "pk": 10 - }, - { - "fields": { - "compounddatatype": 7, - "max_row": null, - "min_row": null, - "transf_xput": 13 - }, - "model": "transformation.xputstructure", - "pk": 11 - }, - { - "fields": { - "compounddatatype": 7, - "max_row": null, - "min_row": null, - "transf_xput": 14 - }, - "model": "transformation.xputstructure", - "pk": 12 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 15 - }, - "model": "transformation.xputstructure", - "pk": 13 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 17 - }, - "model": "transformation.xputstructure", - "pk": 15 - }, - { - "fields": { - "compounddatatype": 7, - "max_row": null, - "min_row": null, - "transf_xput": 18 - }, - "model": "transformation.xputstructure", - "pk": 16 - }, - { - "fields": { - "compounddatatype": 7, - "max_row": null, - "min_row": null, - "transf_xput": 19 - }, - "model": "transformation.xputstructure", - "pk": 17 - }, - { - "fields": { - "compounddatatype": 7, - "max_row": null, - "min_row": null, - "transf_xput": 20 - }, - "model": "transformation.xputstructure", - "pk": 18 - }, - { - "fields": { - "compounddatatype": 7, - "max_row": null, - "min_row": null, - "transf_xput": 22 - }, - "model": "transformation.xputstructure", - "pk": 20 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "strings", - "transformation": 1 - }, - "model": "transformation.transformationinput", - "pk": 1 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "strings", - "transformation": 2 - }, - "model": "transformation.transformationinput", - "pk": 3 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "raw", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 5 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "words_to_reverse", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 7 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "words_to_rereverse", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 9 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "words", - "transformation": 6 - }, - "model": "transformation.transformationinput", - "pk": 11 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "backwords", - "transformation": 7 - }, - "model": "transformation.transformationinput", - "pk": 13 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "lab_data", - "transformation": 8 - }, - "model": "transformation.transformationinput", - "pk": 15 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "words_to_reverse", - "transformation": 9 - }, - "model": "transformation.transformationinput", - "pk": 18 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "data", - "transformation": 10 - }, - "model": "transformation.transformationinput", - "pk": 20 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "same_strings", - "transformation": 1 - }, - "model": "transformation.transformationoutput", - "pk": 2 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "untouched_strings", - "transformation": 2 - }, - "model": "transformation.transformationoutput", - "pk": 4 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "same_raw", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 6 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "reversed_words", - "transformation": 4 - }, - "model": "transformation.transformationoutput", - "pk": 8 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "rereversed_words", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 10 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "more_words", - "transformation": 6 - }, - "model": "transformation.transformationoutput", - "pk": 12 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "more_backwords", - "transformation": 7 - }, - "model": "transformation.transformationoutput", - "pk": 14 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "complemented_lab_data", - "transformation": 8 - }, - "model": "transformation.transformationoutput", - "pk": 17 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "reversed_words", - "transformation": 9 - }, - "model": "transformation.transformationoutput", - "pk": 19 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "unchanged_data", - "transformation": 10 - }, - "model": "transformation.transformationoutput", - "pk": 22 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:00Z", - "email": "bob@talabs.com", - "first_name": "", - "groups": [ - 1 - ], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$WQxc5CtRqLOw$zHTr10E7u1XXD2l17JinBbsdudH8ZQyZkSsWYt9Qsxs=", - "user_permissions": [], - "username": "bob" - }, - "model": "auth.user", - "pk": 2 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:00Z", - "description": "sequences of ASCII characters", - "groups_allowed": [ - 1 - ], - "name": "my_string", - "prototype": null, - "restricts": [ - 1 - ], - "user": 2, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 8 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 6 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 7 - }, - { - "fields": { - "MD5_checksum": "092ef90fe0201f4b002cd200f7754d45", - "_redacted": false, - "dataset_file": "Datasets/2017_04/tmp8oRBAN", - "date_created": "2000-01-01T00:00:01Z", - "description": "blahblahblah", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T01:00:04.513Z", - "name": "blahblah", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "49d48fef110525c0ea4cd811111934ff", - "_redacted": false, - "dataset_file": "Datasets/2017_04/tmpkYOWIM", - "date_created": "2000-01-01T00:00:01Z", - "description": "random reversed words", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T01:00:05.009Z", - "name": "wordbacks", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "17bfea23f36dea9f3f68df25c1b1a392", - "_redacted": false, - "dataset_file": "Datasets/2017_04/tmpIq8uCy", - "date_created": "2000-01-01T00:00:01Z", - "description": "random reversed words", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T01:00:05.114Z", - "name": "backwords", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 3 - }, - { - "fields": { - "description": "a script to do nothing", - "filename": "noop.sh", - "groups_allowed": [ - 1 - ], - "name": "noop", - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 1 - }, - { - "fields": { - "description": "a script to reverse lines of a file", - "filename": "reverse.py", - "groups_allowed": [ - 1 - ], - "name": "reverse", - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "d218c00725d473639f347d456de381d8", - "coderesource": 1, - "content_file": "CodeResources/fdopen", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "1", - "revision_number": 1, - "revision_parent": null, - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "55d9a706ac81b733f851a279d27d8cf2", - "coderesource": 2, - "content_file": "CodeResources/fdopen_RUL0eE8", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "1", - "revision_number": 1, - "revision_parent": null, - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 2 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 1 - }, - { - "fields": { - "driver": 1, - "family": 2, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 2 - }, - { - "fields": { - "driver": 1, - "family": 3, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 3 - }, - { - "fields": { - "driver": 2, - "family": 4, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 4 - }, - { - "fields": { - "driver": 2, - "family": 5, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 5 - }, - { - "fields": { - "driver": 1, - "family": 6, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 6 - }, - { - "fields": { - "driver": 1, - "family": 7, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 7 - }, - { - "fields": { - "description": "a method to do nothing to strings", - "groups_allowed": [ - 1 - ], - "name": "string noop", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 1 - }, - { - "fields": { - "description": "a TOTALLY DIFFERENT method that TOTALLY does SOMETHING to strings by leaving them alone", - "groups_allowed": [ - 1 - ], - "name": "string trivial", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 2 - }, - { - "fields": { - "description": "do nothing to raw data", - "groups_allowed": [ - 1 - ], - "name": "raw noop", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 3 - }, - { - "fields": { - "description": "a method to reverse strings", - "groups_allowed": [ - 1 - ], - "name": "string reverse", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 4 - }, - { - "fields": { - "description": "a method to re-reverse strings", - "groups_allowed": [ - 1 - ], - "name": "string re-reverse", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 5 - }, - { - "fields": { - "description": "a method to do nothing on two columns (word, drow)", - "groups_allowed": [ - 1 - ], - "name": "noop wordback", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 6 - }, - { - "fields": { - "description": "a method to do nothing on two columns", - "groups_allowed": [ - 1 - ], - "name": "noop backword", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 7 - }, - { - "fields": { - "description": "a simple, one-step pipeline", - "groups_allowed": [ - 1 - ], - "name": "simple pipeline", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 1 - }, - { - "fields": { - "description": "a two-step pipeline with custom cable wires at each step", - "groups_allowed": [ - 1 - ], - "name": "two-step pipeline", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 2 - }, - { - "fields": { - "description": "a pipeline with a sub-pipeline", - "groups_allowed": [ - 1 - ], - "name": "nested pipeline", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 3 - }, - { - "fields": { - "family": 1, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 8 - }, - { - "fields": { - "family": 2, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 9 - }, - { - "fields": { - "family": 3, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 10 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 1 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 2 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 3 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 4 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 6 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 7 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:01Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 8 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 9 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 10 - }, - { - "fields": { - "dataset": 1, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 3, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 3 - } -] \ No newline at end of file diff --git a/kive/portal/fixtures/removal.json b/kive/portal/fixtures/removal.json deleted file mode 100644 index 36af76e30..000000000 --- a/kive/portal/fixtures/removal.json +++ /dev/null @@ -1,1660 +0,0 @@ -[ - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "sequence", - "compounddatatype": 5, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 7 - }, - { - "fields": { - "dataset": 1, - "index": 1, - "run": 1 - }, - "model": "archive.runinput", - "pk": 1 - }, - { - "fields": { - "dataset": 1, - "index": 1, - "run": 2 - }, - "model": "archive.runinput", - "pk": 2 - }, - { - "fields": { - "dataset": 3, - "index": 1, - "run": 3 - }, - "model": "archive.runinput", - "pk": 3 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:13Z", - "execrecord": 2, - "reused": false, - "start_time": "2000-01-01T00:00:01Z" - }, - "model": "archive.runcomponent", - "pk": 1 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:01Z", - "execrecord": 1, - "reused": false, - "start_time": "2000-01-01T00:00:01Z" - }, - "model": "archive.runcomponent", - "pk": 2 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:15Z", - "execrecord": 3, - "reused": false, - "start_time": "2000-01-01T00:00:15Z" - }, - "model": "archive.runcomponent", - "pk": 3 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:17Z", - "execrecord": 2, - "reused": true, - "start_time": "2000-01-01T00:00:17Z" - }, - "model": "archive.runcomponent", - "pk": 4 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:17Z", - "execrecord": 1, - "reused": true, - "start_time": "2000-01-01T00:00:17Z" - }, - "model": "archive.runcomponent", - "pk": 5 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:17Z", - "execrecord": 3, - "reused": true, - "start_time": "2000-01-01T00:00:17Z" - }, - "model": "archive.runcomponent", - "pk": 6 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:33Z", - "execrecord": 5, - "reused": false, - "start_time": "2000-01-01T00:00:18Z" - }, - "model": "archive.runcomponent", - "pk": 7 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:48Z", - "execrecord": 7, - "reused": false, - "start_time": "2000-01-01T00:00:34Z" - }, - "model": "archive.runcomponent", - "pk": 8 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:18Z", - "execrecord": 4, - "reused": false, - "start_time": "2000-01-01T00:00:18Z" - }, - "model": "archive.runcomponent", - "pk": 9 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:35Z", - "execrecord": 6, - "reused": false, - "start_time": "2000-01-01T00:00:34Z" - }, - "model": "archive.runcomponent", - "pk": 10 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:50Z", - "execrecord": 8, - "reused": false, - "start_time": "2000-01-01T00:00:49Z" - }, - "model": "archive.runcomponent", - "pk": 11 - }, - { - "fields": { - "pipelinestep": 1, - "run": 1 - }, - "model": "archive.runstep", - "pk": 1 - }, - { - "fields": { - "pipelinestep": 1, - "run": 2 - }, - "model": "archive.runstep", - "pk": 4 - }, - { - "fields": { - "pipelinestep": 4, - "run": 3 - }, - "model": "archive.runstep", - "pk": 7 - }, - { - "fields": { - "pipelinestep": 5, - "run": 3 - }, - "model": "archive.runstep", - "pk": 8 - }, - { - "fields": { - "PSIC": 1, - "dest_runstep": 1 - }, - "model": "archive.runsic", - "pk": 2 - }, - { - "fields": { - "PSIC": 1, - "dest_runstep": 4 - }, - "model": "archive.runsic", - "pk": 5 - }, - { - "fields": { - "PSIC": 6, - "dest_runstep": 7 - }, - "model": "archive.runsic", - "pk": 9 - }, - { - "fields": { - "PSIC": 7, - "dest_runstep": 8 - }, - "model": "archive.runsic", - "pk": 10 - }, - { - "fields": { - "pipelineoutputcable": 2, - "run": 1 - }, - "model": "archive.runoutputcable", - "pk": 3 - }, - { - "fields": { - "pipelineoutputcable": 2, - "run": 2 - }, - "model": "archive.runoutputcable", - "pk": 6 - }, - { - "fields": { - "pipelineoutputcable": 8, - "run": 3 - }, - "model": "archive.runoutputcable", - "pk": 11 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:01Z", - "invoking_record": 2, - "record": 2, - "start_time": "2000-01-01T00:00:01Z" - }, - "model": "archive.execlog", - "pk": 1 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:02Z", - "invoking_record": 1, - "record": 1, - "start_time": "2000-01-01T00:00:01Z" - }, - "model": "archive.execlog", - "pk": 2 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:15Z", - "invoking_record": 3, - "record": 3, - "start_time": "2000-01-01T00:00:15Z" - }, - "model": "archive.execlog", - "pk": 3 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:18Z", - "invoking_record": 9, - "record": 9, - "start_time": "2000-01-01T00:00:18Z" - }, - "model": "archive.execlog", - "pk": 4 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:19Z", - "invoking_record": 7, - "record": 7, - "start_time": "2000-01-01T00:00:19Z" - }, - "model": "archive.execlog", - "pk": 5 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:34Z", - "invoking_record": 10, - "record": 10, - "start_time": "2000-01-01T00:00:34Z" - }, - "model": "archive.execlog", - "pk": 6 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:35Z", - "invoking_record": 8, - "record": 8, - "start_time": "2000-01-01T00:00:35Z" - }, - "model": "archive.execlog", - "pk": 7 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:49Z", - "invoking_record": 11, - "record": 11, - "start_time": "2000-01-01T00:00:49Z" - }, - "model": "archive.execlog", - "pk": 8 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step1_stderr_slurmIDJ_nodeN.txt", - "error_redacted": false, - "execlog": 2, - "install_failed": false, - "output_log": "Logs/step1_stdout_slurmIDJ_nodeN.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 1 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step1_stderr_slurmIDJ_nodeN_j8MbNmb.txt", - "error_redacted": false, - "execlog": 5, - "install_failed": false, - "output_log": "Logs/step1_stdout_slurmIDJ_nodeN_bTBVxxZ.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 2 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step2_stderr_slurmIDJ_nodeN.txt", - "error_redacted": false, - "execlog": 7, - "install_failed": false, - "output_log": "Logs/step2_stdout_slurmIDJ_nodeN.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 5, - "dataset": 1, - "num_rows": 7 - }, - "model": "librarian.datasetstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 5, - "dataset": 2, - "num_rows": 7 - }, - "model": "librarian.datasetstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 5, - "dataset": 3, - "num_rows": 4 - }, - "model": "librarian.datasetstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 5, - "dataset": 4, - "num_rows": 4 - }, - "model": "librarian.datasetstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 5, - "dataset": 5, - "num_rows": 4 - }, - "model": "librarian.datasetstructure", - "pk": 5 - }, - { - "fields": { - "generator": 1 - }, - "model": "librarian.execrecord", - "pk": 1 - }, - { - "fields": { - "generator": 2 - }, - "model": "librarian.execrecord", - "pk": 2 - }, - { - "fields": { - "generator": 3 - }, - "model": "librarian.execrecord", - "pk": 3 - }, - { - "fields": { - "generator": 4 - }, - "model": "librarian.execrecord", - "pk": 4 - }, - { - "fields": { - "generator": 5 - }, - "model": "librarian.execrecord", - "pk": 5 - }, - { - "fields": { - "generator": 6 - }, - "model": "librarian.execrecord", - "pk": 6 - }, - { - "fields": { - "generator": 7 - }, - "model": "librarian.execrecord", - "pk": 7 - }, - { - "fields": { - "generator": 8 - }, - "model": "librarian.execrecord", - "pk": 8 - }, - { - "fields": { - "dataset": 1, - "execrecord": 1, - "generic_input": 5 - }, - "model": "librarian.execrecordin", - "pk": 1 - }, - { - "fields": { - "dataset": 1, - "execrecord": 2, - "generic_input": 1 - }, - "model": "librarian.execrecordin", - "pk": 2 - }, - { - "fields": { - "dataset": 2, - "execrecord": 3, - "generic_input": 2 - }, - "model": "librarian.execrecordin", - "pk": 3 - }, - { - "fields": { - "dataset": 3, - "execrecord": 4, - "generic_input": 10 - }, - "model": "librarian.execrecordin", - "pk": 4 - }, - { - "fields": { - "dataset": 3, - "execrecord": 5, - "generic_input": 1 - }, - "model": "librarian.execrecordin", - "pk": 5 - }, - { - "fields": { - "dataset": 4, - "execrecord": 6, - "generic_input": 2 - }, - "model": "librarian.execrecordin", - "pk": 6 - }, - { - "fields": { - "dataset": 4, - "execrecord": 7, - "generic_input": 1 - }, - "model": "librarian.execrecordin", - "pk": 7 - }, - { - "fields": { - "dataset": 5, - "execrecord": 8, - "generic_input": 2 - }, - "model": "librarian.execrecordin", - "pk": 8 - }, - { - "fields": { - "dataset": 1, - "execrecord": 1, - "generic_output": 1 - }, - "model": "librarian.execrecordout", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "execrecord": 2, - "generic_output": 2 - }, - "model": "librarian.execrecordout", - "pk": 2 - }, - { - "fields": { - "dataset": 2, - "execrecord": 3, - "generic_output": 6 - }, - "model": "librarian.execrecordout", - "pk": 3 - }, - { - "fields": { - "dataset": 3, - "execrecord": 4, - "generic_output": 1 - }, - "model": "librarian.execrecordout", - "pk": 4 - }, - { - "fields": { - "dataset": 4, - "execrecord": 5, - "generic_output": 2 - }, - "model": "librarian.execrecordout", - "pk": 5 - }, - { - "fields": { - "dataset": 4, - "execrecord": 6, - "generic_output": 1 - }, - "model": "librarian.execrecordout", - "pk": 6 - }, - { - "fields": { - "dataset": 5, - "execrecord": 7, - "generic_output": 2 - }, - "model": "librarian.execrecordout", - "pk": 7 - }, - { - "fields": { - "dataset": 5, - "execrecord": 8, - "generic_output": 11 - }, - "model": "librarian.execrecordout", - "pk": 8 - }, - { - "fields": { - "filename": "", - "method": 2, - "path": "", - "requirement": 1 - }, - "model": "method.methoddependency", - "pk": 1 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 3, - "step_num": 1, - "transformation": 1, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 1 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 4, - "step_num": 1, - "transformation": 3, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 2 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 4, - "step_num": 2, - "transformation": 3, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 3 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 1, - "transformation": 1, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 4 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 2, - "transformation": 1, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 5 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 1 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 2 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 3 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 4 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 5 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 6 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 7 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 8 - }, - { - "fields": { - "dest": 1, - "keep_output": false, - "pipelinestep": 1, - "source": 5, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 1 - }, - { - "fields": { - "dest": 5, - "keep_output": false, - "pipelinestep": 2, - "source": 7, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 3 - }, - { - "fields": { - "dest": 5, - "keep_output": false, - "pipelinestep": 3, - "source": 6, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 4 - }, - { - "fields": { - "dest": 1, - "keep_output": false, - "pipelinestep": 4, - "source": 10, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 6 - }, - { - "fields": { - "dest": 1, - "keep_output": false, - "pipelinestep": 5, - "source": 2, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 7 - }, - { - "fields": { - "output_cdt": 5, - "output_idx": 1, - "output_name": "noop_pipeline_out", - "pipeline": 3, - "source": 2, - "source_step": 1 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 2 - }, - { - "fields": { - "output_cdt": 5, - "output_idx": 1, - "output_name": "nested_out", - "pipeline": 4, - "source": 6, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 5 - }, - { - "fields": { - "output_cdt": 5, - "output_idx": 1, - "output_name": "noop_pipeline_out", - "pipeline": 5, - "source": 2, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 8 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 1 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 2 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 3 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 4 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 5 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 6 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 7 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 9 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 10 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 11 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 1 - }, - "model": "transformation.xputstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 2 - }, - "model": "transformation.xputstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 5 - }, - "model": "transformation.xputstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 6 - }, - "model": "transformation.xputstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 7 - }, - "model": "transformation.xputstructure", - "pk": 5 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 9 - }, - "model": "transformation.xputstructure", - "pk": 7 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 10 - }, - "model": "transformation.xputstructure", - "pk": 8 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 11 - }, - "model": "transformation.xputstructure", - "pk": 9 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "nuc_seq_in", - "transformation": 1 - }, - "model": "transformation.transformationinput", - "pk": 1 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "nuc_seq_in", - "transformation": 2 - }, - "model": "transformation.transformationinput", - "pk": 3 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "noop_pipeline_in", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 5 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "nested_in", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 7 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "noop_pipeline_in", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 10 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "nuc_seq_out", - "transformation": 1 - }, - "model": "transformation.transformationoutput", - "pk": 2 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "nuc_seq_out", - "transformation": 2 - }, - "model": "transformation.transformationoutput", - "pk": 4 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "noop_pipeline_out", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 6 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "nested_out", - "transformation": 4 - }, - "model": "transformation.transformationoutput", - "pk": 9 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "noop_pipeline_out", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 11 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:00Z", - "email": "rem@over.sucks", - "first_name": "", - "groups": [ - 1 - ], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$zI3F8CvPlld8$TFFnC8BIyQUFZAiZM6reDZh7a6QiRefICvxtGKDNP8w=", - "user_permissions": [], - "username": "RemOver" - }, - "model": "auth.user", - "pk": 2 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:00Z", - "description": "Sequences of A, C, G, and T", - "groups_allowed": [], - "name": "Nucleotide sequence", - "prototype": null, - "restricts": [ - 1 - ], - "user": 2, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 8 - }, - { - "fields": { - "groups_allowed": [], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 5 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:16Z", - "groups_allowed": [], - "name": "", - "parent_runstep": null, - "paused_by": null, - "pipeline": 3, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "/home/rliang/Documents/Kive/media/Sandboxes/userRemOver_run1_JgLFzl", - "start_time": "2000-01-01T00:00:01Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:53:40.691Z", - "user": 2, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 1 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:17Z", - "groups_allowed": [], - "name": "", - "parent_runstep": null, - "paused_by": null, - "pipeline": 3, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "/home/rliang/Documents/Kive/media/Sandboxes/userRemOver_run2_gR3mvH", - "start_time": "2000-01-01T00:00:17Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:53:56.504Z", - "user": 2, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 2 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:51Z", - "groups_allowed": [], - "name": "", - "parent_runstep": null, - "paused_by": null, - "pipeline": 5, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "/home/rliang/Documents/Kive/media/Sandboxes/userRemOver_run3_PbOjKZ", - "start_time": "2000-01-01T00:00:18Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:53:57.793Z", - "user": 2, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 3 - }, - { - "fields": { - "MD5_checksum": "97aac6a9fbc85d01b9b56d54cd2a5a1a", - "_redacted": false, - "dataset_file": "Datasets/2017_04/tmpqZ8D6n", - "date_created": "2000-01-01T00:00:00Z", - "description": "A dataset for use in the removal test case.", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [], - "last_time_checked": "2017-04-25T00:53:39.674Z", - "name": "Removal test data", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "97aac6a9fbc85d01b9b56d54cd2a5a1a", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step1_nuc_seq_out.csv", - "date_created": "2000-01-01T00:00:12Z", - "description": "Generated data from a run of pipeline \"Nucleotide Sequence Noop:1 (v1)\" started at 2017-04-25 00:53:40.805529+00:00 by RemOver\nrun: 1\nuser: RemOver\nstep: 1\noutput: nuc_seq_out", - "external_path": "", - "externalfiledirectory": null, - "file_source": 1, - "groups_allowed": [], - "last_time_checked": "2017-04-25T00:53:52.450Z", - "name": "run1_step1_outputnuc_seq_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "acf721d5ab57259504ea42bff552a8ab", - "_redacted": false, - "dataset_file": "Datasets/2017_04/tmpvkZRGF", - "date_created": "2000-01-01T00:00:18Z", - "description": "A dataset for use in the removal test case with the two-step Pipeline.", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [], - "last_time_checked": "2017-04-25T00:53:57.730Z", - "name": "Removal test data for a two-step Pipeline", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 3 - }, - { - "fields": { - "MD5_checksum": "acf721d5ab57259504ea42bff552a8ab", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step1_nuc_seq_out_HYoJxmD.csv", - "date_created": "2000-01-01T00:00:32Z", - "description": "Generated data from a run of pipeline \"Nucleotide Sequence two-step Noop:1 (v1)\" started at 2017-04-25 00:53:57.932456+00:00 by RemOver\nrun: 3\nuser: RemOver\nstep: 1\noutput: nuc_seq_out", - "external_path": "", - "externalfiledirectory": null, - "file_source": 7, - "groups_allowed": [], - "last_time_checked": "2017-04-25T00:54:11.862Z", - "name": "run3_step1_outputnuc_seq_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 4 - }, - { - "fields": { - "MD5_checksum": "acf721d5ab57259504ea42bff552a8ab", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step2_nuc_seq_out.csv", - "date_created": "2000-01-01T00:00:47Z", - "description": "Generated data from a run of pipeline \"Nucleotide Sequence two-step Noop:1 (v1)\" started at 2017-04-25 00:53:57.932456+00:00 by RemOver\nrun: 3\nuser: RemOver\nstep: 2\noutput: nuc_seq_out", - "external_path": "", - "externalfiledirectory": null, - "file_source": 8, - "groups_allowed": [], - "last_time_checked": "2017-04-25T00:54:26.982Z", - "name": "run3_step2_outputnuc_seq_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 5 - }, - { - "fields": { - "description": "A noop script that simply writes its input to its output.", - "filename": "noop.bash", - "groups_allowed": [], - "name": "Noop", - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 1 - }, - { - "fields": { - "description": "A script that does nothing to its input and passes it through untouched.", - "filename": "passthrough.bash", - "groups_allowed": [], - "name": "Pass Through", - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "314ec18b433b478ad55d3e1306586816", - "coderesource": 1, - "content_file": "CodeResources/fdopen", - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "1", - "revision_number": 1, - "revision_parent": null, - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "83626e348ac566a61cab9d94683446c7", - "coderesource": 2, - "content_file": "CodeResources/fdopen_7oYknyp", - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "1", - "revision_number": 1, - "revision_parent": null, - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 2 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 1 - }, - { - "fields": { - "driver": 2, - "family": 2, - "groups_allowed": [], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 2 - }, - { - "fields": { - "description": "A noop on nucleotide sequences", - "groups_allowed": [], - "name": "Noop (nucleotide sequence)", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 1 - }, - { - "fields": { - "description": "A pass-through on raw data that uses noop as a dependency", - "groups_allowed": [], - "name": "Pass-through (raw)", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 2 - }, - { - "fields": { - "description": "A noop pipeline for nucleotide sequences.", - "groups_allowed": [], - "name": "Nucleotide Sequence Noop", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 1 - }, - { - "fields": { - "description": "Pipeline with one nested level", - "groups_allowed": [], - "name": "Nested pipeline", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 2 - }, - { - "fields": { - "description": "A two-step noop pipeline for nucleotide sequences.", - "groups_allowed": [], - "name": "Nucleotide Sequence two-step Noop", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 3 - }, - { - "fields": { - "family": 1, - "groups_allowed": [], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 3 - }, - { - "fields": { - "family": 2, - "groups_allowed": [], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 4 - }, - { - "fields": { - "family": 3, - "groups_allowed": [], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 1 - }, - { - "fields": { - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 2 - }, - { - "fields": { - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 3 - }, - { - "fields": { - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 4 - }, - { - "fields": { - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:17Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 5 - }, - { - "fields": { - "dataset": 1, - "end_time": "2000-01-01T00:00:00Z", - "execlog": null, - "start_time": "2000-01-01T00:00:00Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "end_time": "2000-01-01T00:00:13Z", - "execlog": 2, - "start_time": "2000-01-01T00:00:13Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 3, - "end_time": "2000-01-01T00:00:18Z", - "execlog": null, - "start_time": "2000-01-01T00:00:18Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 3 - }, - { - "fields": { - "dataset": 4, - "end_time": "2000-01-01T00:00:33Z", - "execlog": 5, - "start_time": "2000-01-01T00:00:32Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 4 - }, - { - "fields": { - "dataset": 5, - "end_time": "2000-01-01T00:00:48Z", - "execlog": 7, - "start_time": "2000-01-01T00:00:48Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 5 - }, - { - "fields": { - "dataset": 1, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "read_failed": false, - "runcomponent": 2, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.integritychecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "end_time": "2000-01-01T00:00:13Z", - "execlog": 2, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:13Z", - "user": 2 - }, - "model": "datachecking.integritychecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 3, - "end_time": "2000-01-01T00:00:18Z", - "execlog": null, - "read_failed": false, - "runcomponent": 9, - "start_time": "2000-01-01T00:00:18Z", - "user": 2 - }, - "model": "datachecking.integritychecklog", - "pk": 3 - }, - { - "fields": { - "dataset": 4, - "end_time": "2000-01-01T00:00:32Z", - "execlog": 5, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:32Z", - "user": 2 - }, - "model": "datachecking.integritychecklog", - "pk": 4 - }, - { - "fields": { - "dataset": 5, - "end_time": "2000-01-01T00:00:48Z", - "execlog": 7, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:48Z", - "user": 2 - }, - "model": "datachecking.integritychecklog", - "pk": 5 - } -] \ No newline at end of file diff --git a/kive/portal/fixtures/restore_reusable_dataset.json b/kive/portal/fixtures/restore_reusable_dataset.json deleted file mode 100644 index 7d1f92031..000000000 --- a/kive/portal/fixtures/restore_reusable_dataset.json +++ /dev/null @@ -1,1151 +0,0 @@ -[ - { - "fields": { - "dataset": 1, - "index": 1, - "run": 1 - }, - "model": "archive.runinput", - "pk": 1 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:11Z", - "execrecord": 2, - "reused": false, - "start_time": "2000-01-01T00:00:01Z" - }, - "model": "archive.runcomponent", - "pk": 1 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:24Z", - "execrecord": 4, - "reused": false, - "start_time": "2000-01-01T00:00:13Z" - }, - "model": "archive.runcomponent", - "pk": 2 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:01Z", - "execrecord": 1, - "reused": false, - "start_time": "2000-01-01T00:00:01Z" - }, - "model": "archive.runcomponent", - "pk": 3 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:13Z", - "execrecord": 3, - "reused": false, - "start_time": "2000-01-01T00:00:13Z" - }, - "model": "archive.runcomponent", - "pk": 4 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:26Z", - "execrecord": 5, - "reused": false, - "start_time": "2000-01-01T00:00:25Z" - }, - "model": "archive.runcomponent", - "pk": 5 - }, - { - "fields": { - "pipelinestep": 1, - "run": 1 - }, - "model": "archive.runstep", - "pk": 1 - }, - { - "fields": { - "pipelinestep": 2, - "run": 1 - }, - "model": "archive.runstep", - "pk": 2 - }, - { - "fields": { - "PSIC": 1, - "dest_runstep": 1 - }, - "model": "archive.runsic", - "pk": 3 - }, - { - "fields": { - "PSIC": 2, - "dest_runstep": 2 - }, - "model": "archive.runsic", - "pk": 4 - }, - { - "fields": { - "pipelineoutputcable": 3, - "run": 1 - }, - "model": "archive.runoutputcable", - "pk": 5 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:01Z", - "invoking_record": 3, - "record": 3, - "start_time": "2000-01-01T00:00:01Z" - }, - "model": "archive.execlog", - "pk": 1 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:01Z", - "invoking_record": 1, - "record": 1, - "start_time": "2000-01-01T00:00:01Z" - }, - "model": "archive.execlog", - "pk": 2 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:13Z", - "invoking_record": 4, - "record": 4, - "start_time": "2000-01-01T00:00:13Z" - }, - "model": "archive.execlog", - "pk": 3 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:14Z", - "invoking_record": 2, - "record": 2, - "start_time": "2000-01-01T00:00:13Z" - }, - "model": "archive.execlog", - "pk": 4 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:25Z", - "invoking_record": 5, - "record": 5, - "start_time": "2000-01-01T00:00:25Z" - }, - "model": "archive.execlog", - "pk": 5 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step1_stderr_slurmIDJ_nodeN.txt", - "error_redacted": false, - "execlog": 2, - "install_failed": false, - "output_log": "Logs/step1_stdout_slurmIDJ_nodeN.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 1 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step2_stderr_slurmIDJ_nodeN.txt", - "error_redacted": false, - "execlog": 4, - "install_failed": false, - "output_log": "Logs/step2_stdout_slurmIDJ_nodeN.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 2 - }, - { - "fields": { - "generator": 1 - }, - "model": "librarian.execrecord", - "pk": 1 - }, - { - "fields": { - "generator": 2 - }, - "model": "librarian.execrecord", - "pk": 2 - }, - { - "fields": { - "generator": 3 - }, - "model": "librarian.execrecord", - "pk": 3 - }, - { - "fields": { - "generator": 4 - }, - "model": "librarian.execrecord", - "pk": 4 - }, - { - "fields": { - "generator": 5 - }, - "model": "librarian.execrecord", - "pk": 5 - }, - { - "fields": { - "dataset": 1, - "execrecord": 1, - "generic_input": 7 - }, - "model": "librarian.execrecordin", - "pk": 1 - }, - { - "fields": { - "dataset": 1, - "execrecord": 2, - "generic_input": 1 - }, - "model": "librarian.execrecordin", - "pk": 2 - }, - { - "fields": { - "dataset": 2, - "execrecord": 3, - "generic_input": 2 - }, - "model": "librarian.execrecordin", - "pk": 3 - }, - { - "fields": { - "dataset": 2, - "execrecord": 4, - "generic_input": 3 - }, - "model": "librarian.execrecordin", - "pk": 4 - }, - { - "fields": { - "dataset": 3, - "execrecord": 5, - "generic_input": 4 - }, - "model": "librarian.execrecordin", - "pk": 5 - }, - { - "fields": { - "dataset": 1, - "execrecord": 1, - "generic_output": 1 - }, - "model": "librarian.execrecordout", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "execrecord": 2, - "generic_output": 2 - }, - "model": "librarian.execrecordout", - "pk": 2 - }, - { - "fields": { - "dataset": 2, - "execrecord": 3, - "generic_output": 3 - }, - "model": "librarian.execrecordout", - "pk": 3 - }, - { - "fields": { - "dataset": 3, - "execrecord": 4, - "generic_output": 4 - }, - "model": "librarian.execrecordout", - "pk": 4 - }, - { - "fields": { - "dataset": 3, - "execrecord": 5, - "generic_output": 8 - }, - "model": "librarian.execrecordout", - "pk": 5 - }, - { - "fields": { - "fill_colour": "", - "name": "sums_and_products", - "outputs_to_delete": [ - 2 - ], - "pipeline": 4, - "step_num": 1, - "transformation": 1, - "x": 0.4, - "y": 0.4 - }, - "model": "pipeline.pipelinestep", - "pk": 1 - }, - { - "fields": { - "fill_colour": "", - "name": "total_sums", - "outputs_to_delete": [], - "pipeline": 4, - "step_num": 2, - "transformation": 2, - "x": 0.6, - "y": 0.6 - }, - "model": "pipeline.pipelinestep", - "pk": 2 - }, - { - "fields": { - "fill_colour": "", - "name": "sums_and_products", - "outputs_to_delete": [ - 2 - ], - "pipeline": 5, - "step_num": 1, - "transformation": 1, - "x": 0.285714285714286, - "y": 0.285714285714286 - }, - "model": "pipeline.pipelinestep", - "pk": 3 - }, - { - "fields": { - "fill_colour": "", - "name": "total_sums", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 2, - "transformation": 2, - "x": 0.428571428571429, - "y": 0.428571428571429 - }, - "model": "pipeline.pipelinestep", - "pk": 4 - }, - { - "fields": { - "fill_colour": "", - "name": "total_products", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 3, - "transformation": 3, - "x": 0.714285714285714, - "y": 0.714285714285714 - }, - "model": "pipeline.pipelinestep", - "pk": 5 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 1 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 2 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 3 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 4 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 5 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 6 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 7 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 8 - }, - { - "fields": { - "dest": 1, - "keep_output": false, - "pipelinestep": 1, - "source": 7, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 1 - }, - { - "fields": { - "dest": 3, - "keep_output": false, - "pipelinestep": 2, - "source": 2, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 2 - }, - { - "fields": { - "dest": 1, - "keep_output": false, - "pipelinestep": 3, - "source": 9, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 4 - }, - { - "fields": { - "dest": 3, - "keep_output": false, - "pipelinestep": 4, - "source": 2, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 5 - }, - { - "fields": { - "dest": 5, - "keep_output": false, - "pipelinestep": 5, - "source": 2, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 6 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 1, - "output_name": "total_sums", - "pipeline": 4, - "source": 4, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 3 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 1, - "output_name": "total_sums", - "pipeline": 5, - "source": 4, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 7 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 2, - "output_name": "total_products", - "pipeline": 5, - "source": 6, - "source_step": 3 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 8 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 1 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 2 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 3 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 4 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 5 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 6 - }, - { - "fields": { - "x": 0.2, - "y": 0.2 - }, - "model": "transformation.transformationxput", - "pk": 7 - }, - { - "fields": { - "x": 0.8, - "y": 0.8 - }, - "model": "transformation.transformationxput", - "pk": 8 - }, - { - "fields": { - "x": 0.142857142857143, - "y": 0.142857142857143 - }, - "model": "transformation.transformationxput", - "pk": 9 - }, - { - "fields": { - "x": 0.571428571428571, - "y": 0.571428571428571 - }, - "model": "transformation.transformationxput", - "pk": 10 - }, - { - "fields": { - "x": 0.857142857142857, - "y": 0.857142857142857 - }, - "model": "transformation.transformationxput", - "pk": 11 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "pairs", - "transformation": 1 - }, - "model": "transformation.transformationinput", - "pk": 1 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "sums_and_products", - "transformation": 2 - }, - "model": "transformation.transformationinput", - "pk": 3 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "sums_and_products", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 5 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "pairs", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 7 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "pairs", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 9 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "sums_and_products", - "transformation": 1 - }, - "model": "transformation.transformationoutput", - "pk": 2 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "total_sums", - "transformation": 2 - }, - "model": "transformation.transformationoutput", - "pk": 4 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "total_products", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 6 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "total_sums", - "transformation": 4 - }, - "model": "transformation.transformationoutput", - "pk": 8 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "total_sums", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 10 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "total_products", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 11 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:27Z", - "groups_allowed": [], - "name": "", - "parent_runstep": null, - "paused_by": null, - "pipeline": 4, - "priority": 0, - "purged": true, - "runbatch": null, - "sandbox_path": "/home/rliang/Documents/Kive/media/Sandboxes/userkive_run1_NI801W", - "start_time": "2000-01-01T00:00:01Z", - "stopped_by": null, - "time_queued": "2017-04-25T01:00:10.181Z", - "user": 1, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "e093c9eac5d0cae832d4e2071ccfdc5f", - "_redacted": false, - "dataset_file": "Datasets/2017_04/pairs.csv", - "date_created": "2000-01-01T00:00:00Z", - "description": "", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T01:00:10.167Z", - "name": "pairs", - "user": 1, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "44c997f011190635c1e8e618b330a452", - "_redacted": false, - "dataset_file": "", - "date_created": "2000-01-01T00:00:11Z", - "description": "Generated data from a run of pipeline \"sums and products:1 (sums only)\" started at 2017-04-25 01:00:10.287539+00:00 by kive\nrun: 1\nuser: kive\nstep: 1\noutput: sums_and_products", - "external_path": "", - "externalfiledirectory": null, - "file_source": 1, - "groups_allowed": [], - "last_time_checked": "2017-04-25T01:00:20.441Z", - "name": "run1_step1_outputsums_and_products", - "user": 1, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "e237ca8bb33ea0dd3f8d6bb8d1ede977", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step2_total_sums.raw", - "date_created": "2000-01-01T00:00:23Z", - "description": "Generated data from a run of pipeline \"sums and products:1 (sums only)\" started at 2017-04-25 01:00:10.287539+00:00 by kive\nrun: 1\nuser: kive\nstep: 2\noutput: total_sums", - "external_path": "", - "externalfiledirectory": null, - "file_source": 2, - "groups_allowed": [], - "last_time_checked": "2017-04-25T01:00:32.762Z", - "name": "run1_step2_outputtotal_sums", - "user": 1, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 3 - }, - { - "fields": { - "description": "", - "filename": "sums_and_products.py", - "groups_allowed": [ - 1 - ], - "name": "sums_and_products", - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 1 - }, - { - "fields": { - "description": "", - "filename": "total_sums.py", - "groups_allowed": [ - 1 - ], - "name": "total_sums", - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 2 - }, - { - "fields": { - "description": "", - "filename": "total_products.py", - "groups_allowed": [ - 1 - ], - "name": "total_products", - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 3 - }, - { - "fields": { - "MD5_checksum": "47ce4dafc561b053378491af90eb93b9", - "coderesource": 1, - "content_file": "CodeResources/sums_and_products.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "", - "revision_name": "", - "revision_number": 1, - "revision_parent": null, - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "b1a966dbc9cec1fe2dc88ef156b70f01", - "coderesource": 2, - "content_file": "CodeResources/total_sums.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "", - "revision_name": "", - "revision_number": 1, - "revision_parent": null, - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "85bb2c51fa3720a5ff1aba0631e19d25", - "coderesource": 3, - "content_file": "CodeResources/total_products.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "", - "revision_name": "", - "revision_number": 1, - "revision_parent": null, - "user": 1, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 3 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "reusable": 2, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 1 - }, - { - "fields": { - "driver": 2, - "family": 2, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 2 - }, - { - "fields": { - "driver": 3, - "family": 3, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 3 - }, - { - "fields": { - "description": "", - "groups_allowed": [ - 1 - ], - "name": "sums_and_products", - "user": 1, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 1 - }, - { - "fields": { - "description": "", - "groups_allowed": [ - 1 - ], - "name": "total_sums", - "user": 1, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 2 - }, - { - "fields": { - "description": "", - "groups_allowed": [ - 1 - ], - "name": "total_products", - "user": 1, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 3 - }, - { - "fields": { - "description": "", - "groups_allowed": [ - 1 - ], - "name": "sums and products", - "user": 1, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 1 - }, - { - "fields": { - "family": 1, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 4 - }, - { - "fields": { - "family": 1, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 2, - "revision_parent": 4, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "", - "revision_name": "first", - "user": 1, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 1 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "", - "revision_name": "first", - "user": 1, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 2 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "", - "revision_name": "first", - "user": 1, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 3 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "", - "revision_name": "sums only", - "user": 1, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 4 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "", - "revision_name": "sums and products", - "user": 1, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 5 - }, - { - "fields": { - "dataset": 2, - "end_time": "2000-01-01T00:00:11Z", - "execlog": 2, - "start_time": "2000-01-01T00:00:11Z", - "user": 1 - }, - "model": "datachecking.contentchecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 3, - "end_time": "2000-01-01T00:00:24Z", - "execlog": 4, - "start_time": "2000-01-01T00:00:24Z", - "user": 1 - }, - "model": "datachecking.contentchecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 1, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "read_failed": false, - "runcomponent": 3, - "start_time": "2000-01-01T00:00:01Z", - "user": 1 - }, - "model": "datachecking.integritychecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "end_time": "2000-01-01T00:00:11Z", - "execlog": 2, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:11Z", - "user": 1 - }, - "model": "datachecking.integritychecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 3, - "end_time": "2000-01-01T00:00:24Z", - "execlog": 4, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:24Z", - "user": 1 - }, - "model": "datachecking.integritychecklog", - "pk": 3 - } -] diff --git a/kive/portal/fixtures/run_api_tests.json b/kive/portal/fixtures/run_api_tests.json deleted file mode 100644 index 1d8e865dc..000000000 --- a/kive/portal/fixtures/run_api_tests.json +++ /dev/null @@ -1,1528 +0,0 @@ -[ - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "pX_a", - "compounddatatype": 5, - "datatype": 4 - }, - "model": "metadata.compounddatatypemember", - "pk": 7 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "pX_b", - "compounddatatype": 5, - "datatype": 4 - }, - "model": "metadata.compounddatatypemember", - "pk": 8 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "pX_c", - "compounddatatype": 5, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 9 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "a", - "compounddatatype": 6, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 10 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "b", - "compounddatatype": 6, - "datatype": 4 - }, - "model": "metadata.compounddatatypemember", - "pk": 11 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "c", - "compounddatatype": 7, - "datatype": 4 - }, - "model": "metadata.compounddatatypemember", - "pk": 12 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "d", - "compounddatatype": 7, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 13 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "c", - "compounddatatype": 8, - "datatype": 4 - }, - "model": "metadata.compounddatatypemember", - "pk": 14 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "d", - "compounddatatype": 8, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 15 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "e", - "compounddatatype": 8, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 16 - }, - { - "fields": { - "dataset": 1, - "index": 1, - "run": 1 - }, - "model": "archive.runinput", - "pk": 1 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:32Z", - "execrecord": 2, - "reused": false, - "start_time": "2000-01-01T00:00:00Z" - }, - "model": "archive.runcomponent", - "pk": 1 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:45Z", - "execrecord": 4, - "reused": false, - "start_time": "2000-01-01T00:00:01Z" - }, - "model": "archive.runcomponent", - "pk": 2 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:10Z", - "execrecord": 1, - "reused": false, - "start_time": "2000-01-01T00:00:00Z" - }, - "model": "archive.runcomponent", - "pk": 3 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:22Z", - "execrecord": 1, - "reused": false, - "start_time": "2000-01-01T00:00:01Z" - }, - "model": "archive.runcomponent", - "pk": 4 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:34Z", - "execrecord": 3, - "reused": false, - "start_time": "2000-01-01T00:00:34Z" - }, - "model": "archive.runcomponent", - "pk": 5 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:58Z", - "execrecord": 5, - "reused": false, - "start_time": "2000-01-01T00:00:47Z" - }, - "model": "archive.runcomponent", - "pk": 6 - }, - { - "fields": { - "pipelinestep": 3, - "run": 1 - }, - "model": "archive.runstep", - "pk": 1 - }, - { - "fields": { - "pipelinestep": 4, - "run": 1 - }, - "model": "archive.runstep", - "pk": 2 - }, - { - "fields": { - "PSIC": 5, - "dest_runstep": 1 - }, - "model": "archive.runsic", - "pk": 3 - }, - { - "fields": { - "PSIC": 6, - "dest_runstep": 2 - }, - "model": "archive.runsic", - "pk": 4 - }, - { - "fields": { - "pipelineoutputcable": 7, - "run": 1 - }, - "model": "archive.runoutputcable", - "pk": 5 - }, - { - "fields": { - "pipelineoutputcable": 8, - "run": 1 - }, - "model": "archive.runoutputcable", - "pk": 6 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:10Z", - "invoking_record": 3, - "record": 3, - "start_time": "2000-01-01T00:00:01Z" - }, - "model": "archive.execlog", - "pk": 1 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:11Z", - "invoking_record": 1, - "record": 1, - "start_time": "2000-01-01T00:00:11Z" - }, - "model": "archive.execlog", - "pk": 2 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:22Z", - "invoking_record": 4, - "record": 4, - "start_time": "2000-01-01T00:00:11Z" - }, - "model": "archive.execlog", - "pk": 3 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:23Z", - "invoking_record": 2, - "record": 2, - "start_time": "2000-01-01T00:00:22Z" - }, - "model": "archive.execlog", - "pk": 4 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:34Z", - "invoking_record": 5, - "record": 5, - "start_time": "2000-01-01T00:00:34Z" - }, - "model": "archive.execlog", - "pk": 5 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:57Z", - "invoking_record": 6, - "record": 6, - "start_time": "2000-01-01T00:00:47Z" - }, - "model": "archive.execlog", - "pk": 6 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step1_stderr_slurmIDJ_nodeN.txt", - "error_redacted": false, - "execlog": 2, - "install_failed": false, - "output_log": "Logs/step1_stdout_slurmIDJ_nodeN.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 1 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step2_stderr_slurmIDJ_nodeN.txt", - "error_redacted": false, - "execlog": 4, - "install_failed": false, - "output_log": "Logs/step2_stdout_slurmIDJ_nodeN.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 5, - "dataset": 1, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 6, - "dataset": 3, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 7, - "dataset": 4, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 7, - "dataset": 5, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 8, - "dataset": 6, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 5 - }, - { - "fields": { - "generator": 1 - }, - "model": "librarian.execrecord", - "pk": 1 - }, - { - "fields": { - "generator": 2 - }, - "model": "librarian.execrecord", - "pk": 2 - }, - { - "fields": { - "generator": 5 - }, - "model": "librarian.execrecord", - "pk": 3 - }, - { - "fields": { - "generator": 4 - }, - "model": "librarian.execrecord", - "pk": 4 - }, - { - "fields": { - "generator": 6 - }, - "model": "librarian.execrecord", - "pk": 5 - }, - { - "fields": { - "dataset": 1, - "execrecord": 1, - "generic_input": 9 - }, - "model": "librarian.execrecordin", - "pk": 1 - }, - { - "fields": { - "dataset": 3, - "execrecord": 2, - "generic_input": 1 - }, - "model": "librarian.execrecordin", - "pk": 2 - }, - { - "fields": { - "dataset": 4, - "execrecord": 3, - "generic_input": 2 - }, - "model": "librarian.execrecordin", - "pk": 3 - }, - { - "fields": { - "dataset": 3, - "execrecord": 4, - "generic_input": 1 - }, - "model": "librarian.execrecordin", - "pk": 4 - }, - { - "fields": { - "dataset": 5, - "execrecord": 5, - "generic_input": 2 - }, - "model": "librarian.execrecordin", - "pk": 5 - }, - { - "fields": { - "dataset": 3, - "execrecord": 1, - "generic_output": 1 - }, - "model": "librarian.execrecordout", - "pk": 1 - }, - { - "fields": { - "dataset": 4, - "execrecord": 2, - "generic_output": 2 - }, - "model": "librarian.execrecordout", - "pk": 2 - }, - { - "fields": { - "dataset": 4, - "execrecord": 3, - "generic_output": 10 - }, - "model": "librarian.execrecordout", - "pk": 3 - }, - { - "fields": { - "dataset": 5, - "execrecord": 4, - "generic_output": 2 - }, - "model": "librarian.execrecordout", - "pk": 4 - }, - { - "fields": { - "dataset": 6, - "execrecord": 5, - "generic_output": 11 - }, - "model": "librarian.execrecordout", - "pk": 5 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 2, - "step_num": 1, - "transformation": 1, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 1 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 3, - "step_num": 1, - "transformation": 4, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 2 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 1, - "transformation": 1, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 3 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 2, - "transformation": 1, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 4 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 1 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 2 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 3 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 4 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 5 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 6 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 7 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 8 - }, - { - "fields": { - "dest": 1, - "keep_output": false, - "pipelinestep": 1, - "source": 3, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 1 - }, - { - "fields": { - "dest": 5, - "keep_output": false, - "pipelinestep": 2, - "source": 7, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 3 - }, - { - "fields": { - "dest": 1, - "keep_output": false, - "pipelinestep": 3, - "source": 9, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 5 - }, - { - "fields": { - "dest": 1, - "keep_output": false, - "pipelinestep": 4, - "source": 9, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 6 - }, - { - "fields": { - "cable": 1, - "dest_pin": 11, - "source_pin": 8 - }, - "model": "pipeline.customcablewire", - "pk": 1 - }, - { - "fields": { - "cable": 1, - "dest_pin": 10, - "source_pin": 9 - }, - "model": "pipeline.customcablewire", - "pk": 2 - }, - { - "fields": { - "cable": 5, - "dest_pin": 11, - "source_pin": 8 - }, - "model": "pipeline.customcablewire", - "pk": 3 - }, - { - "fields": { - "cable": 5, - "dest_pin": 10, - "source_pin": 9 - }, - "model": "pipeline.customcablewire", - "pk": 4 - }, - { - "fields": { - "cable": 6, - "dest_pin": 11, - "source_pin": 8 - }, - "model": "pipeline.customcablewire", - "pk": 5 - }, - { - "fields": { - "cable": 6, - "dest_pin": 10, - "source_pin": 9 - }, - "model": "pipeline.customcablewire", - "pk": 6 - }, - { - "fields": { - "cable": 8, - "dest_pin": 14, - "source_pin": 12 - }, - "model": "pipeline.customcablewire", - "pk": 7 - }, - { - "fields": { - "cable": 8, - "dest_pin": 15, - "source_pin": 13 - }, - "model": "pipeline.customcablewire", - "pk": 8 - }, - { - "fields": { - "cable": 8, - "dest_pin": 16, - "source_pin": 13 - }, - "model": "pipeline.customcablewire", - "pk": 9 - }, - { - "fields": { - "output_cdt": 7, - "output_idx": 1, - "output_name": "pX_out", - "pipeline": 2, - "source": 2, - "source_step": 1 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 2 - }, - { - "fields": { - "output_cdt": 7, - "output_idx": 1, - "output_name": "pX_out", - "pipeline": 3, - "source": 6, - "source_step": 1 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 4 - }, - { - "fields": { - "output_cdt": 7, - "output_idx": 1, - "output_name": "pX_out_1", - "pipeline": 5, - "source": 2, - "source_step": 1 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 7 - }, - { - "fields": { - "output_cdt": 8, - "output_idx": 2, - "output_name": "pX_out_2", - "pipeline": 5, - "source": 2, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 8 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 1 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 2 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 3 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 4 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 5 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 6 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 7 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 8 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 9 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 10 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 11 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 1 - }, - "model": "transformation.xputstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 7, - "max_row": null, - "min_row": null, - "transf_xput": 2 - }, - "model": "transformation.xputstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 3 - }, - "model": "transformation.xputstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 7, - "max_row": null, - "min_row": null, - "transf_xput": 4 - }, - "model": "transformation.xputstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 7, - "max_row": null, - "min_row": null, - "transf_xput": 6 - }, - "model": "transformation.xputstructure", - "pk": 5 - }, - { - "fields": { - "compounddatatype": 7, - "max_row": null, - "min_row": null, - "transf_xput": 8 - }, - "model": "transformation.xputstructure", - "pk": 6 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 9 - }, - "model": "transformation.xputstructure", - "pk": 7 - }, - { - "fields": { - "compounddatatype": 7, - "max_row": null, - "min_row": null, - "transf_xput": 10 - }, - "model": "transformation.xputstructure", - "pk": 8 - }, - { - "fields": { - "compounddatatype": 8, - "max_row": null, - "min_row": null, - "transf_xput": 11 - }, - "model": "transformation.xputstructure", - "pk": 9 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "mA_in", - "transformation": 1 - }, - "model": "transformation.transformationinput", - "pk": 1 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "pX_in", - "transformation": 2 - }, - "model": "transformation.transformationinput", - "pk": 3 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "mA_in", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 5 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "pX_in", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 7 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "pX_in", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 9 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "mA_out", - "transformation": 1 - }, - "model": "transformation.transformationoutput", - "pk": 2 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "pX_out", - "transformation": 2 - }, - "model": "transformation.transformationoutput", - "pk": 4 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "mA_out", - "transformation": 4 - }, - "model": "transformation.transformationoutput", - "pk": 6 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "pX_out", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 8 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "pX_out_1", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 10 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "pX_out_2", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 11 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:00Z", - "email": "lennon@thebeatles.com", - "first_name": "", - "groups": [ - 1 - ], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$FtzvHn2moQVG$uwmbFeyzsnSO6bpxrsWB+ehb/A5ukzd+/zf7hZF0YSI=", - "user_permissions": [], - "username": "john" - }, - "model": "auth.user", - "pk": 2 - }, - { - "fields": { - "groups_allowed": [], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 6 - }, - { - "fields": { - "groups_allowed": [], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 7 - }, - { - "fields": { - "groups_allowed": [], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 8 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:58Z", - "groups_allowed": [], - "name": "", - "parent_runstep": null, - "paused_by": null, - "pipeline": 5, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "/home/rliang/Documents/Kive/media/Sandboxes/userjohn_run1_FS4JDw", - "start_time": "2000-01-01T00:00:00Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:54:34.636Z", - "user": 2, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "4ce50c714420255e82dd3965ccec07ec", - "_redacted": false, - "dataset_file": "Datasets/2017_04/input_for_test_C_twostep_with_subpipeline.csv", - "date_created": "2000-01-01T00:00:00Z", - "description": "input to pipeline pX", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [], - "last_time_checked": "2017-04-25T00:54:34.174Z", - "name": "pX_in_dataset", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "4ce50c714420255e82dd3965ccec07ec", - "_redacted": false, - "dataset_file": "Datasets/2017_04/input_for_test_C_twostep_with_subpipeline_7rD7T0u.csv", - "date_created": "2000-01-01T00:00:00Z", - "description": "input to pipeline pX", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [], - "last_time_checked": "2017-04-25T00:54:34.258Z", - "name": "pX_in_dataset", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "59c1180b30e68d7013f73b9f63a3adca", - "_redacted": false, - "dataset_file": "", - "date_created": "2000-01-01T00:00:10Z", - "description": "Generated data from a run of pipeline \"self.pf:3 (pX_revision_2)\" started at 2017-04-25 00:54:34.753941+00:00 by john\nrun: 1\nuser: john\nstep: 1\ninput: mA_in", - "external_path": "", - "externalfiledirectory": null, - "file_source": 3, - "groups_allowed": [], - "last_time_checked": "2017-04-25T00:54:44.323Z", - "name": "run1_step1_input1", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 3 - }, - { - "fields": { - "MD5_checksum": "bc2ee5c7dfc2f2b9c93124581fedb96c", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step1_mA_out.csv", - "date_created": "2000-01-01T00:00:31Z", - "description": "Generated data from a run of pipeline \"self.pf:3 (pX_revision_2)\" started at 2017-04-25 00:54:34.753941+00:00 by john\nrun: 1\nuser: john\nstep: 1\noutput: mA_out", - "external_path": "", - "externalfiledirectory": null, - "file_source": 1, - "groups_allowed": [], - "last_time_checked": "2017-04-25T00:55:05.982Z", - "name": "run1_step1_outputmA_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 4 - }, - { - "fields": { - "MD5_checksum": "bc2ee5c7dfc2f2b9c93124581fedb96c", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step2_mA_out.csv", - "date_created": "2000-01-01T00:00:44Z", - "description": "Generated data from a run of pipeline \"self.pf:3 (pX_revision_2)\" started at 2017-04-25 00:54:34.753941+00:00 by john\nrun: 1\nuser: john\nstep: 2\noutput: mA_out", - "external_path": "", - "externalfiledirectory": null, - "file_source": 2, - "groups_allowed": [], - "last_time_checked": "2017-04-25T00:55:19.011Z", - "name": "run1_step2_outputmA_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 5 - }, - { - "fields": { - "MD5_checksum": "6dfd290a5dad5e8ab1de623257bdf1b5", - "_redacted": false, - "dataset_file": "Datasets/2017_04/run1_pX_out_2.csv", - "date_created": "2000-01-01T00:00:57Z", - "description": "Generated data from a run of pipeline \"self.pf:3 (pX_revision_2)\" started at 2017-04-25 00:54:34.753941+00:00 by john\nrun: 1\nuser: john\noutput: pX_out_2", - "external_path": "", - "externalfiledirectory": null, - "file_source": 6, - "groups_allowed": [], - "last_time_checked": "2017-04-25T00:55:31.393Z", - "name": "run1_output2", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 6 - }, - { - "fields": { - "description": "self.mA_cr desc", - "filename": "mA.py", - "groups_allowed": [], - "name": "mA_CR", - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "e353a62fe8cb6a56d31b8c65d70b217d", - "coderesource": 1, - "content_file": "CodeResources/generic_script.py", - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "desc", - "revision_name": "v1", - "revision_number": 1, - "revision_parent": null, - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 1 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 1 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [], - "reusable": 1, - "revision_number": 2, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 4 - }, - { - "fields": { - "description": "self.mf desc", - "groups_allowed": [], - "name": "self.mf", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 1 - }, - { - "fields": { - "description": "self.pf desc", - "groups_allowed": [], - "name": "self.pf", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 1 - }, - { - "fields": { - "family": 1, - "groups_allowed": [], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 2 - }, - { - "fields": { - "family": 1, - "groups_allowed": [], - "published": false, - "revision_number": 2, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 3 - }, - { - "fields": { - "family": 1, - "groups_allowed": [], - "published": false, - "revision_number": 3, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "mA_desc", - "revision_name": "mA", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 1 - }, - { - "fields": { - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "X", - "revision_name": "pX_revision", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 2 - }, - { - "fields": { - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "X", - "revision_name": "pX_raw", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 3 - }, - { - "fields": { - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "mA_desc", - "revision_name": "mA_raw", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 4 - }, - { - "fields": { - "groups_allowed": [], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "X2", - "revision_name": "pX_revision_2", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 5 - }, - { - "fields": { - "dataset": 1, - "end_time": "2000-01-01T00:00:00Z", - "execlog": null, - "start_time": "2000-01-01T00:00:00Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 3, - "end_time": "2000-01-01T00:00:10Z", - "execlog": 1, - "start_time": "2000-01-01T00:00:10Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 4, - "end_time": "2000-01-01T00:00:32Z", - "execlog": 2, - "start_time": "2000-01-01T00:00:32Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 3 - }, - { - "fields": { - "dataset": 5, - "end_time": "2000-01-01T00:00:45Z", - "execlog": 4, - "start_time": "2000-01-01T00:00:45Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 4 - }, - { - "fields": { - "dataset": 6, - "end_time": "2000-01-01T00:00:58Z", - "execlog": 6, - "start_time": "2000-01-01T00:00:57Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 5 - }, - { - "fields": { - "dataset": 1, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "read_failed": false, - "runcomponent": 3, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.integritychecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 3, - "end_time": "2000-01-01T00:00:22Z", - "execlog": 3, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:22Z", - "user": 2 - }, - "model": "datachecking.integritychecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 4, - "end_time": "2000-01-01T00:00:32Z", - "execlog": 2, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:32Z", - "user": 2 - }, - "model": "datachecking.integritychecklog", - "pk": 3 - }, - { - "fields": { - "dataset": 5, - "end_time": "2000-01-01T00:00:45Z", - "execlog": 4, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:45Z", - "user": 2 - }, - "model": "datachecking.integritychecklog", - "pk": 4 - } -] \ No newline at end of file diff --git a/kive/portal/fixtures/run_component_too_many_checks.json b/kive/portal/fixtures/run_component_too_many_checks.json deleted file mode 100644 index f0278aca3..000000000 --- a/kive/portal/fixtures/run_component_too_many_checks.json +++ /dev/null @@ -1,2138 +0,0 @@ -[ - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "word", - "compounddatatype": 5, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 7 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "word", - "compounddatatype": 6, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 8 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "drow", - "compounddatatype": 6, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 9 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "drow", - "compounddatatype": 7, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 10 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "word", - "compounddatatype": 7, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 11 - }, - { - "fields": { - "dataset": 2, - "index": 1, - "run": 1 - }, - "model": "archive.runinput", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "index": 1, - "run": 2 - }, - "model": "archive.runinput", - "pk": 2 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:13Z", - "execrecord": 2, - "reused": false, - "start_time": "2000-01-01T00:00:02Z" - }, - "model": "archive.runcomponent", - "pk": 1 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:25Z", - "execrecord": 4, - "reused": false, - "start_time": "2000-01-01T00:00:14Z" - }, - "model": "archive.runcomponent", - "pk": 2 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:02Z", - "execrecord": 1, - "reused": false, - "start_time": "2000-01-01T00:00:02Z" - }, - "model": "archive.runcomponent", - "pk": 3 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:15Z", - "execrecord": 3, - "reused": false, - "start_time": "2000-01-01T00:00:14Z" - }, - "model": "archive.runcomponent", - "pk": 4 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:27Z", - "execrecord": 5, - "reused": false, - "start_time": "2000-01-01T00:00:27Z" - }, - "model": "archive.runcomponent", - "pk": 5 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:30Z", - "execrecord": 2, - "reused": true, - "start_time": "2000-01-01T00:00:29Z" - }, - "model": "archive.runcomponent", - "pk": 6 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:59Z", - "execrecord": 6, - "reused": false, - "start_time": "2000-01-01T00:00:30Z" - }, - "model": "archive.runcomponent", - "pk": 7 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:30Z", - "execrecord": 1, - "reused": true, - "start_time": "2000-01-01T00:00:29Z" - }, - "model": "archive.runcomponent", - "pk": 8 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:46Z", - "execrecord": 3, - "reused": false, - "start_time": "2000-01-01T00:00:30Z" - }, - "model": "archive.runcomponent", - "pk": 9 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:01:01Z", - "execrecord": 7, - "reused": false, - "start_time": "2000-01-01T00:01:00Z" - }, - "model": "archive.runcomponent", - "pk": 10 - }, - { - "fields": { - "pipelinestep": 1, - "run": 1 - }, - "model": "archive.runstep", - "pk": 1 - }, - { - "fields": { - "pipelinestep": 2, - "run": 1 - }, - "model": "archive.runstep", - "pk": 2 - }, - { - "fields": { - "pipelinestep": 3, - "run": 2 - }, - "model": "archive.runstep", - "pk": 6 - }, - { - "fields": { - "pipelinestep": 4, - "run": 2 - }, - "model": "archive.runstep", - "pk": 7 - }, - { - "fields": { - "PSIC": 1, - "dest_runstep": 1 - }, - "model": "archive.runsic", - "pk": 3 - }, - { - "fields": { - "PSIC": 2, - "dest_runstep": 2 - }, - "model": "archive.runsic", - "pk": 4 - }, - { - "fields": { - "PSIC": 4, - "dest_runstep": 6 - }, - "model": "archive.runsic", - "pk": 8 - }, - { - "fields": { - "PSIC": 5, - "dest_runstep": 7 - }, - "model": "archive.runsic", - "pk": 9 - }, - { - "fields": { - "pipelineoutputcable": 3, - "run": 1 - }, - "model": "archive.runoutputcable", - "pk": 5 - }, - { - "fields": { - "pipelineoutputcable": 6, - "run": 2 - }, - "model": "archive.runoutputcable", - "pk": 10 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:02Z", - "invoking_record": 3, - "record": 3, - "start_time": "2000-01-01T00:00:02Z" - }, - "model": "archive.execlog", - "pk": 1 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:03Z", - "invoking_record": 1, - "record": 1, - "start_time": "2000-01-01T00:00:03Z" - }, - "model": "archive.execlog", - "pk": 2 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:14Z", - "invoking_record": 4, - "record": 4, - "start_time": "2000-01-01T00:00:14Z" - }, - "model": "archive.execlog", - "pk": 3 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:15Z", - "invoking_record": 2, - "record": 2, - "start_time": "2000-01-01T00:00:15Z" - }, - "model": "archive.execlog", - "pk": 4 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:27Z", - "invoking_record": 5, - "record": 5, - "start_time": "2000-01-01T00:00:27Z" - }, - "model": "archive.execlog", - "pk": 5 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:30Z", - "invoking_record": 7, - "record": 8, - "start_time": "2000-01-01T00:00:30Z" - }, - "model": "archive.execlog", - "pk": 6 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:30Z", - "invoking_record": 7, - "record": 6, - "start_time": "2000-01-01T00:00:30Z" - }, - "model": "archive.execlog", - "pk": 7 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:45Z", - "invoking_record": 9, - "record": 9, - "start_time": "2000-01-01T00:00:45Z" - }, - "model": "archive.execlog", - "pk": 8 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:46Z", - "invoking_record": 7, - "record": 7, - "start_time": "2000-01-01T00:00:46Z" - }, - "model": "archive.execlog", - "pk": 9 - }, - { - "fields": { - "end_time": "2000-01-01T00:01:00Z", - "invoking_record": 10, - "record": 10, - "start_time": "2000-01-01T00:01:00Z" - }, - "model": "archive.execlog", - "pk": 10 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step1_stderr_slurmIDJ_nodeN.txt", - "error_redacted": false, - "execlog": 2, - "install_failed": false, - "output_log": "Logs/step1_stdout_slurmIDJ_nodeN.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 1 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step2_stderr_slurmIDJ_nodeN.txt", - "error_redacted": false, - "execlog": 4, - "install_failed": false, - "output_log": "Logs/step2_stdout_slurmIDJ_nodeN.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 2 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step1_stderr_slurmIDJ_nodeN_K29E8bc.txt", - "error_redacted": false, - "execlog": 7, - "install_failed": false, - "output_log": "Logs/step1_stdout_slurmIDJ_nodeN_rJsXjZp.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 3 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step2_stderr_slurmIDJ_nodeN_j7dv6pe.txt", - "error_redacted": false, - "execlog": 9, - "install_failed": false, - "output_log": "Logs/step2_stdout_slurmIDJ_nodeN_zBGHBo9.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 5, - "dataset": 1, - "num_rows": 1 - }, - "model": "librarian.datasetstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 6, - "dataset": 2, - "num_rows": 20 - }, - "model": "librarian.datasetstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 7, - "dataset": 3, - "num_rows": 20 - }, - "model": "librarian.datasetstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 6, - "dataset": 4, - "num_rows": 20 - }, - "model": "librarian.datasetstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 6, - "dataset": 5, - "num_rows": 20 - }, - "model": "librarian.datasetstructure", - "pk": 5 - }, - { - "fields": { - "compounddatatype": 6, - "dataset": 6, - "num_rows": 20 - }, - "model": "librarian.datasetstructure", - "pk": 6 - }, - { - "fields": { - "generator": 1 - }, - "model": "librarian.execrecord", - "pk": 1 - }, - { - "fields": { - "generator": 2 - }, - "model": "librarian.execrecord", - "pk": 2 - }, - { - "fields": { - "generator": 3 - }, - "model": "librarian.execrecord", - "pk": 3 - }, - { - "fields": { - "generator": 4 - }, - "model": "librarian.execrecord", - "pk": 4 - }, - { - "fields": { - "generator": 5 - }, - "model": "librarian.execrecord", - "pk": 5 - }, - { - "fields": { - "generator": 9 - }, - "model": "librarian.execrecord", - "pk": 6 - }, - { - "fields": { - "generator": 10 - }, - "model": "librarian.execrecord", - "pk": 7 - }, - { - "fields": { - "dataset": 2, - "execrecord": 1, - "generic_input": 15 - }, - "model": "librarian.execrecordin", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "execrecord": 2, - "generic_input": 11 - }, - "model": "librarian.execrecordin", - "pk": 2 - }, - { - "fields": { - "dataset": 4, - "execrecord": 3, - "generic_input": 12 - }, - "model": "librarian.execrecordin", - "pk": 3 - }, - { - "fields": { - "dataset": 4, - "execrecord": 4, - "generic_input": 11 - }, - "model": "librarian.execrecordin", - "pk": 4 - }, - { - "fields": { - "dataset": 5, - "execrecord": 5, - "generic_input": 12 - }, - "model": "librarian.execrecordin", - "pk": 5 - }, - { - "fields": { - "dataset": 4, - "execrecord": 6, - "generic_input": 7 - }, - "model": "librarian.execrecordin", - "pk": 6 - }, - { - "fields": { - "dataset": 6, - "execrecord": 7, - "generic_input": 8 - }, - "model": "librarian.execrecordin", - "pk": 7 - }, - { - "fields": { - "dataset": 2, - "execrecord": 1, - "generic_output": 11 - }, - "model": "librarian.execrecordout", - "pk": 1 - }, - { - "fields": { - "dataset": 4, - "execrecord": 2, - "generic_output": 12 - }, - "model": "librarian.execrecordout", - "pk": 2 - }, - { - "fields": { - "dataset": 4, - "execrecord": 3, - "generic_output": 11 - }, - "model": "librarian.execrecordout", - "pk": 3 - }, - { - "fields": { - "dataset": 5, - "execrecord": 4, - "generic_output": 12 - }, - "model": "librarian.execrecordout", - "pk": 4 - }, - { - "fields": { - "dataset": 5, - "execrecord": 5, - "generic_output": 16 - }, - "model": "librarian.execrecordout", - "pk": 5 - }, - { - "fields": { - "dataset": 6, - "execrecord": 6, - "generic_output": 8 - }, - "model": "librarian.execrecordout", - "pk": 6 - }, - { - "fields": { - "dataset": 6, - "execrecord": 7, - "generic_output": 18 - }, - "model": "librarian.execrecordout", - "pk": 7 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [ - 12 - ], - "pipeline": 8, - "step_num": 1, - "transformation": 6, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 1 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 8, - "step_num": 2, - "transformation": 6, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 2 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [ - 12 - ], - "pipeline": 9, - "step_num": 1, - "transformation": 6, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 3 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 9, - "step_num": 2, - "transformation": 4, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 4 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 1 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 2 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 3 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 4 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 5 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 6 - }, - { - "fields": { - "dest": 11, - "keep_output": false, - "pipelinestep": 1, - "source": 15, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 1 - }, - { - "fields": { - "dest": 11, - "keep_output": false, - "pipelinestep": 2, - "source": 12, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 2 - }, - { - "fields": { - "dest": 11, - "keep_output": false, - "pipelinestep": 3, - "source": 17, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 4 - }, - { - "fields": { - "dest": 7, - "keep_output": false, - "pipelinestep": 4, - "source": 12, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 5 - }, - { - "fields": { - "output_cdt": 6, - "output_idx": 1, - "output_name": "samedata", - "pipeline": 8, - "source": 12, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 3 - }, - { - "fields": { - "output_cdt": 6, - "output_idx": 1, - "output_name": "reversed_data", - "pipeline": 9, - "source": 8, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 6 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 1 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 2 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 3 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 4 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 5 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 6 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 7 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 8 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 9 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 10 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 11 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 12 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 13 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 14 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 15 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 16 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 17 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 18 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 1 - }, - "model": "transformation.xputstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 2 - }, - "model": "transformation.xputstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 3 - }, - "model": "transformation.xputstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 5, - "max_row": null, - "min_row": null, - "transf_xput": 4 - }, - "model": "transformation.xputstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 7 - }, - "model": "transformation.xputstructure", - "pk": 5 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 8 - }, - "model": "transformation.xputstructure", - "pk": 6 - }, - { - "fields": { - "compounddatatype": 7, - "max_row": null, - "min_row": null, - "transf_xput": 9 - }, - "model": "transformation.xputstructure", - "pk": 7 - }, - { - "fields": { - "compounddatatype": 7, - "max_row": null, - "min_row": null, - "transf_xput": 10 - }, - "model": "transformation.xputstructure", - "pk": 8 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 11 - }, - "model": "transformation.xputstructure", - "pk": 9 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 12 - }, - "model": "transformation.xputstructure", - "pk": 10 - }, - { - "fields": { - "compounddatatype": 7, - "max_row": null, - "min_row": null, - "transf_xput": 13 - }, - "model": "transformation.xputstructure", - "pk": 11 - }, - { - "fields": { - "compounddatatype": 7, - "max_row": null, - "min_row": null, - "transf_xput": 14 - }, - "model": "transformation.xputstructure", - "pk": 12 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 15 - }, - "model": "transformation.xputstructure", - "pk": 13 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 16 - }, - "model": "transformation.xputstructure", - "pk": 14 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 17 - }, - "model": "transformation.xputstructure", - "pk": 15 - }, - { - "fields": { - "compounddatatype": 6, - "max_row": null, - "min_row": null, - "transf_xput": 18 - }, - "model": "transformation.xputstructure", - "pk": 16 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "strings", - "transformation": 1 - }, - "model": "transformation.transformationinput", - "pk": 1 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "strings", - "transformation": 2 - }, - "model": "transformation.transformationinput", - "pk": 3 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "raw", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 5 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "words_to_reverse", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 7 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "words_to_rereverse", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 9 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "words", - "transformation": 6 - }, - "model": "transformation.transformationinput", - "pk": 11 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "backwords", - "transformation": 7 - }, - "model": "transformation.transformationinput", - "pk": 13 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "data", - "transformation": 8 - }, - "model": "transformation.transformationinput", - "pk": 15 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "data", - "transformation": 9 - }, - "model": "transformation.transformationinput", - "pk": 17 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "same_strings", - "transformation": 1 - }, - "model": "transformation.transformationoutput", - "pk": 2 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "untouched_strings", - "transformation": 2 - }, - "model": "transformation.transformationoutput", - "pk": 4 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "same_raw", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 6 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "reversed_words", - "transformation": 4 - }, - "model": "transformation.transformationoutput", - "pk": 8 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "rereversed_words", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 10 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "more_words", - "transformation": 6 - }, - "model": "transformation.transformationoutput", - "pk": 12 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "more_backwords", - "transformation": 7 - }, - "model": "transformation.transformationoutput", - "pk": 14 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "samedata", - "transformation": 8 - }, - "model": "transformation.transformationoutput", - "pk": 16 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "reversed_data", - "transformation": 9 - }, - "model": "transformation.transformationoutput", - "pk": 18 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:00Z", - "email": "bob@talabs.com", - "first_name": "", - "groups": [ - 1 - ], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$H3MglPlMqka1$TcqBv43vEHj7GwENe6Y2V/rScv3KK8eQ5iwg9OH7a2o=", - "user_permissions": [], - "username": "bob" - }, - "model": "auth.user", - "pk": 2 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:00Z", - "description": "sequences of ASCII characters", - "groups_allowed": [ - 1 - ], - "name": "my_string", - "prototype": null, - "restricts": [ - 1 - ], - "user": 2, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 8 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 6 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 7 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:28Z", - "groups_allowed": [ - 1 - ], - "name": "", - "parent_runstep": null, - "paused_by": null, - "pipeline": 8, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "/home/rliang/Documents/Kive/media/Sandboxes/userbob_run1_lXFoYJ", - "start_time": "2000-01-01T00:00:02Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:55:37.380Z", - "user": 2, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 1 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:01:02Z", - "groups_allowed": [ - 1 - ], - "name": "", - "parent_runstep": null, - "paused_by": null, - "pipeline": 9, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "/home/rliang/Documents/Kive/media/Sandboxes/userbob_run2_JpKXxu", - "start_time": "2000-01-01T00:00:29Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:56:04.663Z", - "user": 2, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "092ef90fe0201f4b002cd200f7754d45", - "_redacted": false, - "dataset_file": "Datasets/2017_04/tmpfvGizy", - "date_created": "2000-01-01T00:00:01Z", - "description": "blahblahblah", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:55:36.333Z", - "name": "blahblah", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "49d48fef110525c0ea4cd811111934ff", - "_redacted": false, - "dataset_file": "Datasets/2017_04/tmpikmdM0", - "date_created": "2000-01-01T00:00:01Z", - "description": "random reversed words", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:55:36.807Z", - "name": "wordbacks", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "17bfea23f36dea9f3f68df25c1b1a392", - "_redacted": false, - "dataset_file": "Datasets/2017_04/tmpJ1TvFs", - "date_created": "2000-01-01T00:00:01Z", - "description": "random reversed words", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:55:36.908Z", - "name": "backwords", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 3 - }, - { - "fields": { - "MD5_checksum": "49d48fef110525c0ea4cd811111934ff", - "_redacted": false, - "dataset_file": "", - "date_created": "2000-01-01T00:00:12Z", - "description": "Generated data from a run of pipeline \"Two-step pipeline:1 (v1)\" started at 2017-04-25 00:55:37.499097+00:00 by bob\nrun: 1\nuser: bob\nstep: 1\noutput: more_words", - "external_path": "", - "externalfiledirectory": null, - "file_source": 1, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:55:47.672Z", - "name": "run1_step1_outputmore_words", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 4 - }, - { - "fields": { - "MD5_checksum": "49d48fef110525c0ea4cd811111934ff", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step2_more_words.csv", - "date_created": "2000-01-01T00:00:24Z", - "description": "Generated data from a run of pipeline \"Two-step pipeline:1 (v1)\" started at 2017-04-25 00:55:37.499097+00:00 by bob\nrun: 1\nuser: bob\nstep: 2\noutput: more_words", - "external_path": "", - "externalfiledirectory": null, - "file_source": 2, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:56:00.072Z", - "name": "run1_step2_outputmore_words", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 5 - }, - { - "fields": { - "MD5_checksum": "49d48fef110525c0ea4cd811111934ff", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step2_reversed_words.csv", - "date_created": "2000-01-01T00:00:58Z", - "description": "Generated data from a run of pipeline \"Pipeline that will follow the first:1 (v1)\" started at 2017-04-25 00:56:04.784389+00:00 by bob\nrun: 2\nuser: bob\nstep: 2\noutput: reversed_words", - "external_path": "", - "externalfiledirectory": null, - "file_source": 7, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:56:33.495Z", - "name": "run2_step2_outputreversed_words", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 6 - }, - { - "fields": { - "description": "a script to do nothing", - "filename": "noop.sh", - "groups_allowed": [ - 1 - ], - "name": "noop", - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 1 - }, - { - "fields": { - "description": "a script to reverse lines of a file", - "filename": "reverse.py", - "groups_allowed": [ - 1 - ], - "name": "reverse", - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "d218c00725d473639f347d456de381d8", - "coderesource": 1, - "content_file": "CodeResources/fdopen", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "1", - "revision_number": 1, - "revision_parent": null, - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "55d9a706ac81b733f851a279d27d8cf2", - "coderesource": 2, - "content_file": "CodeResources/fdopen_CvmZr7l", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "1", - "revision_number": 1, - "revision_parent": null, - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 2 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 1 - }, - { - "fields": { - "driver": 1, - "family": 2, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 2 - }, - { - "fields": { - "driver": 1, - "family": 3, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 3 - }, - { - "fields": { - "driver": 2, - "family": 4, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 4 - }, - { - "fields": { - "driver": 2, - "family": 5, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 5 - }, - { - "fields": { - "driver": 1, - "family": 6, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 6 - }, - { - "fields": { - "driver": 1, - "family": 7, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 7 - }, - { - "fields": { - "description": "a method to do nothing to strings", - "groups_allowed": [ - 1 - ], - "name": "string noop", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 1 - }, - { - "fields": { - "description": "a TOTALLY DIFFERENT method that TOTALLY does SOMETHING to strings by leaving them alone", - "groups_allowed": [ - 1 - ], - "name": "string trivial", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 2 - }, - { - "fields": { - "description": "do nothing to raw data", - "groups_allowed": [ - 1 - ], - "name": "raw noop", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 3 - }, - { - "fields": { - "description": "a method to reverse strings", - "groups_allowed": [ - 1 - ], - "name": "string reverse", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 4 - }, - { - "fields": { - "description": "a method to re-reverse strings", - "groups_allowed": [ - 1 - ], - "name": "string re-reverse", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 5 - }, - { - "fields": { - "description": "a method to do nothing on two columns (word, drow)", - "groups_allowed": [ - 1 - ], - "name": "noop wordback", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 6 - }, - { - "fields": { - "description": "a method to do nothing on two columns", - "groups_allowed": [ - 1 - ], - "name": "noop backword", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 7 - }, - { - "fields": { - "description": "Toy pipeline for testing data check cleaning of RunSteps.", - "groups_allowed": [ - 1 - ], - "name": "Two-step pipeline", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 1 - }, - { - "fields": { - "description": "Toy pipeline that will need to recover its first step when following the above.", - "groups_allowed": [ - 1 - ], - "name": "Pipeline that will follow the first", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 2 - }, - { - "fields": { - "family": 1, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 8 - }, - { - "fields": { - "family": 2, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 9 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 1 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 2 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 3 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 4 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 6 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 7 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:01Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 8 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:29Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 9 - }, - { - "fields": { - "dataset": 1, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 3, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 3 - }, - { - "fields": { - "dataset": 4, - "end_time": "2000-01-01T00:00:13Z", - "execlog": 2, - "start_time": "2000-01-01T00:00:13Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 4 - }, - { - "fields": { - "dataset": 5, - "end_time": "2000-01-01T00:00:25Z", - "execlog": 4, - "start_time": "2000-01-01T00:00:25Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 5 - }, - { - "fields": { - "dataset": 6, - "end_time": "2000-01-01T00:00:59Z", - "execlog": 9, - "start_time": "2000-01-01T00:00:58Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 6 - }, - { - "fields": { - "dataset": 2, - "end_time": "2000-01-01T00:00:02Z", - "execlog": null, - "read_failed": false, - "runcomponent": 3, - "start_time": "2000-01-01T00:00:02Z", - "user": 2 - }, - "model": "datachecking.integritychecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 4, - "end_time": "2000-01-01T00:00:12Z", - "execlog": 2, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:12Z", - "user": 2 - }, - "model": "datachecking.integritychecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 5, - "end_time": "2000-01-01T00:00:25Z", - "execlog": 4, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:25Z", - "user": 2 - }, - "model": "datachecking.integritychecklog", - "pk": 3 - }, - { - "fields": { - "dataset": 2, - "end_time": "2000-01-01T00:00:30Z", - "execlog": null, - "read_failed": false, - "runcomponent": 8, - "start_time": "2000-01-01T00:00:30Z", - "user": 2 - }, - "model": "datachecking.integritychecklog", - "pk": 4 - }, - { - "fields": { - "dataset": 4, - "end_time": "2000-01-01T00:00:44Z", - "execlog": 7, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:44Z", - "user": 2 - }, - "model": "datachecking.integritychecklog", - "pk": 5 - }, - { - "fields": { - "dataset": 6, - "end_time": "2000-01-01T00:00:58Z", - "execlog": 9, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:58Z", - "user": 2 - }, - "model": "datachecking.integritychecklog", - "pk": 6 - } -] \ No newline at end of file diff --git a/kive/portal/fixtures/run_pipelines_recovering_reused_step.json b/kive/portal/fixtures/run_pipelines_recovering_reused_step.json deleted file mode 100644 index b1578b318..000000000 --- a/kive/portal/fixtures/run_pipelines_recovering_reused_step.json +++ /dev/null @@ -1,3751 +0,0 @@ -[ - { - "fields": { - "datatype": 8, - "rule": "^[ACGTacgt]*$", - "ruletype": "regexp" - }, - "model": "metadata.basicconstraint", - "pk": 6 - }, - { - "fields": { - "datatype": 9, - "rule": "^[ACGUacgu]*$", - "ruletype": "regexp" - }, - "model": "metadata.basicconstraint", - "pk": 7 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "label", - "compounddatatype": 5, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 7 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "integer", - "compounddatatype": 5, - "datatype": 4 - }, - "model": "metadata.compounddatatypemember", - "pk": 8 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "float", - "compounddatatype": 5, - "datatype": 3 - }, - "model": "metadata.compounddatatypemember", - "pk": 9 - }, - { - "fields": { - "blankable": false, - "column_idx": 4, - "column_name": "bool", - "compounddatatype": 5, - "datatype": 2 - }, - "model": "metadata.compounddatatypemember", - "pk": 10 - }, - { - "fields": { - "blankable": false, - "column_idx": 5, - "column_name": "rna", - "compounddatatype": 5, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 11 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "label", - "compounddatatype": 6, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 12 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "label", - "compounddatatype": 7, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 13 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "PBMCseq", - "compounddatatype": 7, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 14 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "PLAseq", - "compounddatatype": 7, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 15 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "SeqToComplement", - "compounddatatype": 8, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 16 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "ComplementedSeq", - "compounddatatype": 9, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 17 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "SeqToComplement", - "compounddatatype": 10, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 18 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "ComplementedSeq", - "compounddatatype": 11, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 19 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "x", - "compounddatatype": 12, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 20 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "y", - "compounddatatype": 12, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 21 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "k", - "compounddatatype": 13, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 22 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "a", - "compounddatatype": 14, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 23 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "b", - "compounddatatype": 14, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 24 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "c", - "compounddatatype": 14, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 25 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "a^2", - "compounddatatype": 15, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 26 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "b^2", - "compounddatatype": 15, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 27 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "c^2", - "compounddatatype": 15, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 28 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "StrCol1", - "compounddatatype": 16, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 29 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "DNACol2", - "compounddatatype": 16, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 30 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "StrCol3", - "compounddatatype": 16, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 31 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "a", - "compounddatatype": 17, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 32 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "b", - "compounddatatype": 17, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 33 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "c", - "compounddatatype": 17, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 34 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "x", - "compounddatatype": 18, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 35 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "y", - "compounddatatype": 18, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 36 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "word", - "compounddatatype": 19, - "datatype": 10 - }, - "model": "metadata.compounddatatypemember", - "pk": 37 - }, - { - "fields": { - "dataset": 18, - "index": 1, - "run": 1 - }, - "model": "archive.runinput", - "pk": 1 - }, - { - "fields": { - "dataset": 18, - "index": 1, - "run": 2 - }, - "model": "archive.runinput", - "pk": 2 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:17Z", - "execrecord": 2, - "reused": false, - "start_time": "2000-01-01T00:00:03Z" - }, - "model": "archive.runcomponent", - "pk": 1 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:32Z", - "execrecord": 4, - "reused": false, - "start_time": "2000-01-01T00:00:19Z" - }, - "model": "archive.runcomponent", - "pk": 2 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:04Z", - "execrecord": 1, - "reused": false, - "start_time": "2000-01-01T00:00:03Z" - }, - "model": "archive.runcomponent", - "pk": 3 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:19Z", - "execrecord": 3, - "reused": false, - "start_time": "2000-01-01T00:00:19Z" - }, - "model": "archive.runcomponent", - "pk": 4 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:34Z", - "execrecord": 5, - "reused": false, - "start_time": "2000-01-01T00:00:33Z" - }, - "model": "archive.runcomponent", - "pk": 5 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:36Z", - "execrecord": 2, - "reused": true, - "start_time": "2000-01-01T00:00:35Z" - }, - "model": "archive.runcomponent", - "pk": 6 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:01:02Z", - "execrecord": 6, - "reused": false, - "start_time": "2000-01-01T00:00:36Z" - }, - "model": "archive.runcomponent", - "pk": 7 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:36Z", - "execrecord": 1, - "reused": true, - "start_time": "2000-01-01T00:00:35Z" - }, - "model": "archive.runcomponent", - "pk": 8 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:50Z", - "execrecord": 3, - "reused": false, - "start_time": "2000-01-01T00:00:36Z" - }, - "model": "archive.runcomponent", - "pk": 9 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:01:04Z", - "execrecord": 7, - "reused": false, - "start_time": "2000-01-01T00:01:04Z" - }, - "model": "archive.runcomponent", - "pk": 10 - }, - { - "fields": { - "pipelinestep": 5, - "run": 1 - }, - "model": "archive.runstep", - "pk": 1 - }, - { - "fields": { - "pipelinestep": 6, - "run": 1 - }, - "model": "archive.runstep", - "pk": 2 - }, - { - "fields": { - "pipelinestep": 7, - "run": 2 - }, - "model": "archive.runstep", - "pk": 6 - }, - { - "fields": { - "pipelinestep": 8, - "run": 2 - }, - "model": "archive.runstep", - "pk": 7 - }, - { - "fields": { - "PSIC": 12, - "dest_runstep": 1 - }, - "model": "archive.runsic", - "pk": 3 - }, - { - "fields": { - "PSIC": 13, - "dest_runstep": 2 - }, - "model": "archive.runsic", - "pk": 4 - }, - { - "fields": { - "PSIC": 15, - "dest_runstep": 6 - }, - "model": "archive.runsic", - "pk": 8 - }, - { - "fields": { - "PSIC": 16, - "dest_runstep": 7 - }, - "model": "archive.runsic", - "pk": 9 - }, - { - "fields": { - "pipelineoutputcable": 14, - "run": 1 - }, - "model": "archive.runoutputcable", - "pk": 5 - }, - { - "fields": { - "pipelineoutputcable": 17, - "run": 2 - }, - "model": "archive.runoutputcable", - "pk": 10 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:04Z", - "invoking_record": 3, - "record": 3, - "start_time": "2000-01-01T00:00:04Z" - }, - "model": "archive.execlog", - "pk": 1 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:04Z", - "invoking_record": 1, - "record": 1, - "start_time": "2000-01-01T00:00:04Z" - }, - "model": "archive.execlog", - "pk": 2 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:19Z", - "invoking_record": 4, - "record": 4, - "start_time": "2000-01-01T00:00:19Z" - }, - "model": "archive.execlog", - "pk": 3 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:19Z", - "invoking_record": 2, - "record": 2, - "start_time": "2000-01-01T00:00:19Z" - }, - "model": "archive.execlog", - "pk": 4 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:33Z", - "invoking_record": 5, - "record": 5, - "start_time": "2000-01-01T00:00:33Z" - }, - "model": "archive.execlog", - "pk": 5 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:36Z", - "invoking_record": 7, - "record": 8, - "start_time": "2000-01-01T00:00:36Z" - }, - "model": "archive.execlog", - "pk": 6 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:36Z", - "invoking_record": 7, - "record": 6, - "start_time": "2000-01-01T00:00:36Z" - }, - "model": "archive.execlog", - "pk": 7 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:50Z", - "invoking_record": 9, - "record": 9, - "start_time": "2000-01-01T00:00:50Z" - }, - "model": "archive.execlog", - "pk": 8 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:50Z", - "invoking_record": 7, - "record": 7, - "start_time": "2000-01-01T00:00:50Z" - }, - "model": "archive.execlog", - "pk": 9 - }, - { - "fields": { - "end_time": "2000-01-01T00:01:04Z", - "invoking_record": 10, - "record": 10, - "start_time": "2000-01-01T00:01:04Z" - }, - "model": "archive.execlog", - "pk": 10 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step1_stderr_slurmIDJ_nodeN.txt", - "error_redacted": false, - "execlog": 2, - "install_failed": false, - "output_log": "Logs/step1_stdout_slurmIDJ_nodeN.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 1 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step2_stderr_slurmIDJ_nodeN.txt", - "error_redacted": false, - "execlog": 4, - "install_failed": false, - "output_log": "Logs/step2_stdout_slurmIDJ_nodeN.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 2 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step1_stderr_slurmIDJ_nodeN_l1xqfJq.txt", - "error_redacted": false, - "execlog": 7, - "install_failed": false, - "output_log": "Logs/step1_stdout_slurmIDJ_nodeN_SRXCGv7.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 3 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step2_stderr_slurmIDJ_nodeN_w081IPd.txt", - "error_redacted": false, - "execlog": 9, - "install_failed": false, - "output_log": "Logs/step2_stdout_slurmIDJ_nodeN_VdSh1ZP.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 14, - "dataset": 1, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 2, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 13, - "dataset": 3, - "num_rows": 13 - }, - "model": "librarian.datasetstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 13, - "dataset": 4, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 6, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 5 - }, - { - "fields": { - "compounddatatype": 14, - "dataset": 7, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 6 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 8, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 7 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 9, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 8 - }, - { - "fields": { - "compounddatatype": 13, - "dataset": 10, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 9 - }, - { - "fields": { - "compounddatatype": 14, - "dataset": 13, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 10 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 14, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 11 - }, - { - "fields": { - "compounddatatype": 17, - "dataset": 15, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 12 - }, - { - "fields": { - "compounddatatype": 18, - "dataset": 16, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 13 - }, - { - "fields": { - "compounddatatype": 18, - "dataset": 17, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 14 - }, - { - "fields": { - "compounddatatype": 19, - "dataset": 18, - "num_rows": 1 - }, - "model": "librarian.datasetstructure", - "pk": 15 - }, - { - "fields": { - "compounddatatype": 19, - "dataset": 19, - "num_rows": 1 - }, - "model": "librarian.datasetstructure", - "pk": 16 - }, - { - "fields": { - "compounddatatype": 19, - "dataset": 20, - "num_rows": 1 - }, - "model": "librarian.datasetstructure", - "pk": 17 - }, - { - "fields": { - "compounddatatype": 19, - "dataset": 21, - "num_rows": 1 - }, - "model": "librarian.datasetstructure", - "pk": 18 - }, - { - "fields": { - "generator": 1 - }, - "model": "librarian.execrecord", - "pk": 1 - }, - { - "fields": { - "generator": 2 - }, - "model": "librarian.execrecord", - "pk": 2 - }, - { - "fields": { - "generator": 3 - }, - "model": "librarian.execrecord", - "pk": 3 - }, - { - "fields": { - "generator": 4 - }, - "model": "librarian.execrecord", - "pk": 4 - }, - { - "fields": { - "generator": 5 - }, - "model": "librarian.execrecord", - "pk": 5 - }, - { - "fields": { - "generator": 9 - }, - "model": "librarian.execrecord", - "pk": 6 - }, - { - "fields": { - "generator": 10 - }, - "model": "librarian.execrecord", - "pk": 7 - }, - { - "fields": { - "dataset": 18, - "execrecord": 1, - "generic_input": 26 - }, - "model": "librarian.execrecordin", - "pk": 1 - }, - { - "fields": { - "dataset": 18, - "execrecord": 2, - "generic_input": 20 - }, - "model": "librarian.execrecordin", - "pk": 2 - }, - { - "fields": { - "dataset": 19, - "execrecord": 3, - "generic_input": 21 - }, - "model": "librarian.execrecordin", - "pk": 3 - }, - { - "fields": { - "dataset": 19, - "execrecord": 4, - "generic_input": 20 - }, - "model": "librarian.execrecordin", - "pk": 4 - }, - { - "fields": { - "dataset": 20, - "execrecord": 5, - "generic_input": 21 - }, - "model": "librarian.execrecordin", - "pk": 5 - }, - { - "fields": { - "dataset": 19, - "execrecord": 6, - "generic_input": 22 - }, - "model": "librarian.execrecordin", - "pk": 6 - }, - { - "fields": { - "dataset": 21, - "execrecord": 7, - "generic_input": 23 - }, - "model": "librarian.execrecordin", - "pk": 7 - }, - { - "fields": { - "dataset": 18, - "execrecord": 1, - "generic_output": 20 - }, - "model": "librarian.execrecordout", - "pk": 1 - }, - { - "fields": { - "dataset": 19, - "execrecord": 2, - "generic_output": 21 - }, - "model": "librarian.execrecordout", - "pk": 2 - }, - { - "fields": { - "dataset": 19, - "execrecord": 3, - "generic_output": 20 - }, - "model": "librarian.execrecordout", - "pk": 3 - }, - { - "fields": { - "dataset": 20, - "execrecord": 4, - "generic_output": 21 - }, - "model": "librarian.execrecordout", - "pk": 4 - }, - { - "fields": { - "dataset": 20, - "execrecord": 5, - "generic_output": 28 - }, - "model": "librarian.execrecordout", - "pk": 5 - }, - { - "fields": { - "dataset": 21, - "execrecord": 6, - "generic_output": 23 - }, - "model": "librarian.execrecordout", - "pk": 6 - }, - { - "fields": { - "dataset": 21, - "execrecord": 7, - "generic_output": 31 - }, - "model": "librarian.execrecordout", - "pk": 7 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 4, - "step_num": 1, - "transformation": 2, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 1 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 1, - "transformation": 1, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 2 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 2, - "transformation": 4, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 3 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 3, - "transformation": 3, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 4 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [ - 21 - ], - "pipeline": 9, - "step_num": 1, - "transformation": 6, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 5 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 9, - "step_num": 2, - "transformation": 6, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 6 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [ - 21 - ], - "pipeline": 10, - "step_num": 1, - "transformation": 6, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 7 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 10, - "step_num": 2, - "transformation": 7, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 8 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 1 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 2 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 3 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 4 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 5 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 6 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 7 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 8 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 9 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 10 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 11 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 12 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 13 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 14 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 15 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 16 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 17 - }, - { - "fields": { - "dest": 3, - "keep_output": false, - "pipelinestep": 1, - "source": 11, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 1 - }, - { - "fields": { - "dest": 4, - "keep_output": false, - "pipelinestep": 1, - "source": 12, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 2 - }, - { - "fields": { - "dest": 1, - "keep_output": false, - "pipelinestep": 2, - "source": 15, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 4 - }, - { - "fields": { - "dest": 11, - "keep_output": false, - "pipelinestep": 3, - "source": 13, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 5 - }, - { - "fields": { - "dest": 12, - "keep_output": false, - "pipelinestep": 3, - "source": 14, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 6 - }, - { - "fields": { - "dest": 7, - "keep_output": false, - "pipelinestep": 4, - "source": 2, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 7 - }, - { - "fields": { - "dest": 6, - "keep_output": false, - "pipelinestep": 4, - "source": 16, - "source_step": 2 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 8 - }, - { - "fields": { - "dest": 20, - "keep_output": false, - "pipelinestep": 5, - "source": 26, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 12 - }, - { - "fields": { - "dest": 20, - "keep_output": false, - "pipelinestep": 6, - "source": 21, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 13 - }, - { - "fields": { - "dest": 20, - "keep_output": false, - "pipelinestep": 7, - "source": 29, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 15 - }, - { - "fields": { - "dest": 22, - "keep_output": false, - "pipelinestep": 8, - "source": 21, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 16 - }, - { - "fields": { - "cable": 5, - "dest_pin": 21, - "source_pin": 23 - }, - "model": "pipeline.customcablewire", - "pk": 1 - }, - { - "fields": { - "cable": 5, - "dest_pin": 20, - "source_pin": 25 - }, - "model": "pipeline.customcablewire", - "pk": 2 - }, - { - "fields": { - "cable": 7, - "dest_pin": 21, - "source_pin": 20 - }, - "model": "pipeline.customcablewire", - "pk": 3 - }, - { - "fields": { - "cable": 7, - "dest_pin": 20, - "source_pin": 21 - }, - "model": "pipeline.customcablewire", - "pk": 4 - }, - { - "fields": { - "cable": 9, - "dest_pin": 21, - "source_pin": 24 - }, - "model": "pipeline.customcablewire", - "pk": 5 - }, - { - "fields": { - "cable": 9, - "dest_pin": 20, - "source_pin": 25 - }, - "model": "pipeline.customcablewire", - "pk": 6 - }, - { - "fields": { - "output_cdt": 14, - "output_idx": 1, - "output_name": "D1_out", - "pipeline": 4, - "source": 5, - "source_step": 1 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 3 - }, - { - "fields": { - "output_cdt": 12, - "output_idx": 1, - "output_name": "E1_out", - "pipeline": 5, - "source": 16, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 9 - }, - { - "fields": { - "output_cdt": 13, - "output_idx": 2, - "output_name": "E2_out", - "pipeline": 5, - "source": 8, - "source_step": 3 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 10 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 3, - "output_name": "E3_rawout", - "pipeline": 5, - "source": 10, - "source_step": 3 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 11 - }, - { - "fields": { - "output_cdt": 19, - "output_idx": 1, - "output_name": "p_one_out", - "pipeline": 9, - "source": 21, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 14 - }, - { - "fields": { - "output_cdt": 19, - "output_idx": 1, - "output_name": "p_two_out", - "pipeline": 10, - "source": 23, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 17 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 1 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 2 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 3 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 4 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 5 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 6 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 7 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 8 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 9 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 10 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 11 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 12 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 13 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 14 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 15 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 16 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 17 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 18 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 19 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 20 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 21 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 22 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 23 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 24 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 25 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 26 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 28 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 29 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 31 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 2 - }, - "model": "transformation.xputstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 3 - }, - "model": "transformation.xputstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 4 - }, - "model": "transformation.xputstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": 5, - "min_row": null, - "transf_xput": 5 - }, - "model": "transformation.xputstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": null, - "min_row": null, - "transf_xput": 6 - }, - "model": "transformation.xputstructure", - "pk": 5 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 7 - }, - "model": "transformation.xputstructure", - "pk": 6 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 8 - }, - "model": "transformation.xputstructure", - "pk": 7 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 11 - }, - "model": "transformation.xputstructure", - "pk": 8 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 12 - }, - "model": "transformation.xputstructure", - "pk": 9 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": null, - "min_row": null, - "transf_xput": 13 - }, - "model": "transformation.xputstructure", - "pk": 10 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": 10, - "transf_xput": 14 - }, - "model": "transformation.xputstructure", - "pk": 11 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": 5, - "min_row": null, - "transf_xput": 16 - }, - "model": "transformation.xputstructure", - "pk": 12 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": 5, - "min_row": null, - "transf_xput": 17 - }, - "model": "transformation.xputstructure", - "pk": 13 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 18 - }, - "model": "transformation.xputstructure", - "pk": 14 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 20 - }, - "model": "transformation.xputstructure", - "pk": 15 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 21 - }, - "model": "transformation.xputstructure", - "pk": 16 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 22 - }, - "model": "transformation.xputstructure", - "pk": 17 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 23 - }, - "model": "transformation.xputstructure", - "pk": 18 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 26 - }, - "model": "transformation.xputstructure", - "pk": 19 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 28 - }, - "model": "transformation.xputstructure", - "pk": 21 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 29 - }, - "model": "transformation.xputstructure", - "pk": 22 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 31 - }, - "model": "transformation.xputstructure", - "pk": 24 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "A1_rawin", - "transformation": 1 - }, - "model": "transformation.transformationinput", - "pk": 1 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "B1_in", - "transformation": 2 - }, - "model": "transformation.transformationinput", - "pk": 3 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "B2_in", - "transformation": 2 - }, - "model": "transformation.transformationinput", - "pk": 4 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "C1_in", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 6 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "C2_in", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 7 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "D1_in", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 11 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "D2_in", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 12 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "E1_in", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 13 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "E2_in", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 14 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "E3_rawin", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 15 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "strings", - "transformation": 6 - }, - "model": "transformation.transformationinput", - "pk": 20 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "strings", - "transformation": 7 - }, - "model": "transformation.transformationinput", - "pk": 22 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "raw", - "transformation": 8 - }, - "model": "transformation.transformationinput", - "pk": 24 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "p_one_in", - "transformation": 9 - }, - "model": "transformation.transformationinput", - "pk": 26 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "p_two_in", - "transformation": 10 - }, - "model": "transformation.transformationinput", - "pk": 29 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "A1_out", - "transformation": 1 - }, - "model": "transformation.transformationoutput", - "pk": 2 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "B1_out", - "transformation": 2 - }, - "model": "transformation.transformationoutput", - "pk": 5 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "C1_out", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 8 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "C2_rawout", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 9 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "C3_rawout", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 10 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "D1_out", - "transformation": 4 - }, - "model": "transformation.transformationoutput", - "pk": 16 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "E1_out", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 17 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "E2_out", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 18 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "E3_rawout", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 19 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "same_strings", - "transformation": 6 - }, - "model": "transformation.transformationoutput", - "pk": 21 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "untouched_strings", - "transformation": 7 - }, - "model": "transformation.transformationoutput", - "pk": 23 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "same_raw", - "transformation": 8 - }, - "model": "transformation.transformationoutput", - "pk": 25 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "p_one_out", - "transformation": 9 - }, - "model": "transformation.transformationoutput", - "pk": 28 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "p_two_out", - "transformation": 10 - }, - "model": "transformation.transformationoutput", - "pk": 31 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:00Z", - "email": "lennon@thebeatles.com", - "first_name": "", - "groups": [ - 1 - ], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$Gc550Kd3v3Ad$87jpkokvWmloFlusEyCjMzWaJoIsKy8rMBIgBNBZwDo=", - "user_permissions": [], - "username": "john" - }, - "model": "auth.user", - "pk": 2 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:00Z", - "email": "starr@thebeatles.com", - "first_name": "", - "groups": [], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$tqW94Bf469rV$z053kZ1kDTCsyTMDv0UYrFdPx1Bx7AcXKLbjKpU0wZs=", - "user_permissions": [], - "username": "ringo" - }, - "model": "auth.user", - "pk": 3 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:02Z", - "email": "bob@talabs.com", - "first_name": "", - "groups": [ - 1 - ], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$o0Pv7fWl9WZF$YnqN/wqQ33XNk8bfQLUM4ERO/q/Z/rzGAqXAgQ74DZY=", - "user_permissions": [], - "username": "bob" - }, - "model": "auth.user", - "pk": 4 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:00Z", - "description": "String consisting of ACGTacgt", - "groups_allowed": [ - 1 - ], - "name": "DNANucSeq", - "prototype": null, - "restricts": [ - 1 - ], - "user": 2, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 8 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:00Z", - "description": "String consisting of ACGUacgu", - "groups_allowed": [ - 1 - ], - "name": "RNANucSeq", - "prototype": null, - "restricts": [ - 1 - ], - "user": 2, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 9 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:02Z", - "description": "sequences of ASCII characters", - "groups_allowed": [ - 1 - ], - "name": "my_string", - "prototype": null, - "restricts": [ - 1 - ], - "user": 4, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 10 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [], - "name": "", - "user": 2, - "users_allowed": [ - 3 - ] - }, - "model": "metadata.compounddatatype", - "pk": 6 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 7 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 8 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 9 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 10 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 11 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 12 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 13 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 14 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 15 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 16 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 17 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 18 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 4, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 19 - }, - { - "fields": { - "_runstate": 3, - "description": "Bob runs p_one", - "end_time": "2000-01-01T00:00:35Z", - "groups_allowed": [ - 1 - ], - "name": "RunOne", - "parent_runstep": null, - "paused_by": null, - "pipeline": 9, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "/home/rliang/Documents/Kive/media/Sandboxes/userbob_run1_5XhKrS", - "start_time": "2000-01-01T00:00:03Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:56:44.124Z", - "user": 4, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 1 - }, - { - "fields": { - "_runstate": 3, - "description": "Bob runs p_two, which tries to reuse part of run_one but ultimately needs to recover", - "end_time": "2000-01-01T00:01:05Z", - "groups_allowed": [ - 1 - ], - "name": "RunOne", - "parent_runstep": null, - "paused_by": null, - "pipeline": 10, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "/home/rliang/Documents/Kive/media/Sandboxes/userbob_run2_wRix6h", - "start_time": "2000-01-01T00:00:35Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:57:15.941Z", - "user": 4, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "ab7afe2f453d27fd168641a1f9271f26", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_triplet.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:56:41.739Z", - "name": "triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "02d5b66858d003b81ae3dc712b74e85c", - "_redacted": false, - "dataset_file": "Datasets/2017_04/doublet_cdt.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:56:41.808Z", - "name": "doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "29feb269edcac613d6eea432c14640bd", - "_redacted": false, - "dataset_file": "Datasets/2017_04/singlet_cdt_large.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:56:41.869Z", - "name": "singlet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 3 - }, - { - "fields": { - "MD5_checksum": "00fd50bfe7ac63e2165a442f4101e05c", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_singlet.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:56:41.921Z", - "name": "singlet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 4 - }, - { - "fields": { - "MD5_checksum": "7dc85e11b5c02e434af5bd3b3da9938e", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_raw.fasta", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:56:41.972Z", - "name": "raw_DS", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 5 - }, - { - "fields": { - "MD5_checksum": "542676b23e121d16db8d41ccdae65fd1", - "_redacted": false, - "dataset_file": "", - "date_created": "2000-01-01T00:00:01Z", - "description": "", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:56:42.014Z", - "name": "D1_in_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 6 - }, - { - "fields": { - "MD5_checksum": "56f9e5585b9d699829e2fda931d944f8", - "_redacted": false, - "dataset_file": "Datasets/2017_04/C1_in_triplet.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "triplet 3 rows", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:56:42.064Z", - "name": "C1_in_triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 7 - }, - { - "fields": { - "MD5_checksum": "10a4bd15d1300fcaf5a3bdad54ef1d28", - "_redacted": false, - "dataset_file": "", - "date_created": "2000-01-01T00:00:01Z", - "description": "", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:56:42.133Z", - "name": "C2_in_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 8 - }, - { - "fields": { - "MD5_checksum": "10a4bd15d1300fcaf5a3bdad54ef1d28", - "_redacted": false, - "dataset_file": "Datasets/2017_04/E11_32_output.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "result of E11_32 fed by doublet_cdt.csv", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:56:42.184Z", - "name": "E11_32 output doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 9 - }, - { - "fields": { - "MD5_checksum": "00fd50bfe7ac63e2165a442f4101e05c", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_singlet_0GStQao.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:56:42.243Z", - "name": "raw", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 10 - }, - { - "fields": { - "MD5_checksum": "7dc85e11b5c02e434af5bd3b3da9938e", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_raw_Rz9tFVR.fasta", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:56:42.286Z", - "name": "C2_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 11 - }, - { - "fields": { - "MD5_checksum": "7dc85e11b5c02e434af5bd3b3da9938e", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_raw_zE1x9Hi.fasta", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:56:42.312Z", - "name": "C3_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 12 - }, - { - "fields": { - "MD5_checksum": "56f9e5585b9d699829e2fda931d944f8", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_triplet_3_rows.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:56:42.334Z", - "name": "triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 13 - }, - { - "fields": { - "MD5_checksum": "83957f8aea1c75da9f7fbf9bc90da979", - "_redacted": false, - "dataset_file": "Datasets/2017_04/doublet_remuxed_from_t3r.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "doublet remuxed from triplet", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:56:42.401Z", - "name": "E1_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 14 - }, - { - "fields": { - "MD5_checksum": "d83866e4c23cd9503aa2f9fc565502b5", - "_redacted": false, - "dataset_file": "Datasets/2017_04/DNA_triplet.csv", - "date_created": "2000-01-01T00:00:02Z", - "description": "DNA triplet data", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:56:42.461Z", - "name": "DNA_triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 15 - }, - { - "fields": { - "MD5_checksum": "9eab0dfed5fcc89bd581f61995816ef0", - "_redacted": false, - "dataset_file": "Datasets/2017_04/E01_21_DNA_doublet.csv", - "date_created": "2000-01-01T00:00:02Z", - "description": "DNA doublet data coming from DNA_triplet.csv but remultiplexed according to cable E01_21", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:56:42.599Z", - "name": "E01_21_DNA_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 16 - }, - { - "fields": { - "MD5_checksum": "9b55b9431da6dba74ea61edf8e2ac044", - "_redacted": false, - "dataset_file": "Datasets/2017_04/E21_41_DNA_doublet.csv", - "date_created": "2000-01-01T00:00:02Z", - "description": "DNA doublet data coming from DNA_triplet.csv but remultiplexed according to cable E21_41", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:56:42.701Z", - "name": "E21_41_DNA_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 17 - }, - { - "fields": { - "MD5_checksum": "092ef90fe0201f4b002cd200f7754d45", - "_redacted": false, - "dataset_file": "Datasets/2017_04/tmpGstAHg", - "date_created": "2000-01-01T00:00:03Z", - "description": "blahblahblah", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:56:44.061Z", - "name": "blahblah", - "user": 4, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 18 - }, - { - "fields": { - "MD5_checksum": "092ef90fe0201f4b002cd200f7754d45", - "_redacted": false, - "dataset_file": "", - "date_created": "2000-01-01T00:00:16Z", - "description": "Generated data from a run of pipeline \"p_one:1 (v1)\" started at 2017-04-25 00:56:44.239305+00:00 by bob\nrun: 1\nuser: bob\nstep: 1\noutput: same_strings", - "external_path": "", - "externalfiledirectory": null, - "file_source": 1, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:56:57.193Z", - "name": "run1_step1_outputsame_strings", - "user": 4, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 19 - }, - { - "fields": { - "MD5_checksum": "092ef90fe0201f4b002cd200f7754d45", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step2_same_strings.csv", - "date_created": "2000-01-01T00:00:31Z", - "description": "Generated data from a run of pipeline \"p_one:1 (v1)\" started at 2017-04-25 00:56:44.239305+00:00 by bob\nrun: 1\nuser: bob\nstep: 2\noutput: same_strings", - "external_path": "", - "externalfiledirectory": null, - "file_source": 2, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:57:11.916Z", - "name": "run1_step2_outputsame_strings", - "user": 4, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 20 - }, - { - "fields": { - "MD5_checksum": "092ef90fe0201f4b002cd200f7754d45", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step2_untouched_strings.csv", - "date_created": "2000-01-01T00:01:01Z", - "description": "Generated data from a run of pipeline \"p_two:1 (v1)\" started at 2017-04-25 00:57:16.060392+00:00 by bob\nrun: 2\nuser: bob\nstep: 2\noutput: untouched_strings", - "external_path": "", - "externalfiledirectory": null, - "file_source": 7, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:57:41.852Z", - "name": "run2_step2_outputuntouched_strings", - "user": 4, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 21 - }, - { - "fields": { - "description": "Just a CR", - "filename": "generic_script.py", - "groups_allowed": [ - 1 - ], - "name": "genericCR", - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 1 - }, - { - "fields": { - "description": "a script to do nothing", - "filename": "noop.sh", - "groups_allowed": [ - 1 - ], - "name": "noop", - "user": 4, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "e353a62fe8cb6a56d31b8c65d70b217d", - "coderesource": 1, - "content_file": "CodeResources/generic_script.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "desc", - "revision_name": "v1", - "revision_number": 1, - "revision_parent": null, - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "d218c00725d473639f347d456de381d8", - "coderesource": 2, - "content_file": "CodeResources/fdopen", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "1", - "revision_number": 1, - "revision_parent": null, - "user": 4, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 2 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 1 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 2, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 2 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 3, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 3 - }, - { - "fields": { - "driver": 2, - "family": 2, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 6 - }, - { - "fields": { - "driver": 2, - "family": 3, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 7 - }, - { - "fields": { - "driver": 2, - "family": 4, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 8 - }, - { - "fields": { - "description": "Holds methods A/B/C", - "groups_allowed": [ - 1 - ], - "name": "method_family", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 1 - }, - { - "fields": { - "description": "a method to do nothing to strings", - "groups_allowed": [ - 1 - ], - "name": "string noop", - "user": 4, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 2 - }, - { - "fields": { - "description": "a TOTALLY DIFFERENT method that TOTALLY does SOMETHING to strings by leaving them alone", - "groups_allowed": [ - 1 - ], - "name": "string trivial", - "user": 4, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 3 - }, - { - "fields": { - "description": "do nothing to raw data", - "groups_allowed": [ - 1 - ], - "name": "raw noop", - "user": 4, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 4 - }, - { - "fields": { - "description": "PF desc", - "groups_allowed": [ - 1 - ], - "name": "Pipeline_family", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 1 - }, - { - "fields": { - "description": "two no-ops", - "groups_allowed": [ - 1 - ], - "name": "p_one", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 2 - }, - { - "fields": { - "description": "one no-op then one trivial", - "groups_allowed": [ - 1 - ], - "name": "p_two", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 3 - }, - { - "fields": { - "family": 1, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 4 - }, - { - "fields": { - "family": 1, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 2, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 5 - }, - { - "fields": { - "family": 2, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 9 - }, - { - "fields": { - "family": 3, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 10 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "A_desc", - "revision_name": "mA_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 1 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "B_desc", - "revision_name": "mB_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 2 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "C_desc", - "revision_name": "mC_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 3 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "D", - "revision_name": "pD_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 4 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "E", - "revision_name": "pE_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 4, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 6 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 4, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 7 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 4, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 8 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 9 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:03Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 10 - }, - { - "fields": { - "dataset": 1, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 3, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 3 - }, - { - "fields": { - "dataset": 4, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 4 - }, - { - "fields": { - "dataset": 6, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 5 - }, - { - "fields": { - "dataset": 7, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 6 - }, - { - "fields": { - "dataset": 8, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 7 - }, - { - "fields": { - "dataset": 9, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 8 - }, - { - "fields": { - "dataset": 10, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 9 - }, - { - "fields": { - "dataset": 13, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 10 - }, - { - "fields": { - "dataset": 14, - "end_time": "2000-01-01T00:00:02Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 11 - }, - { - "fields": { - "dataset": 15, - "end_time": "2000-01-01T00:00:02Z", - "execlog": null, - "start_time": "2000-01-01T00:00:02Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 12 - }, - { - "fields": { - "dataset": 16, - "end_time": "2000-01-01T00:00:02Z", - "execlog": null, - "start_time": "2000-01-01T00:00:02Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 13 - }, - { - "fields": { - "dataset": 17, - "end_time": "2000-01-01T00:00:02Z", - "execlog": null, - "start_time": "2000-01-01T00:00:02Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 14 - }, - { - "fields": { - "dataset": 18, - "end_time": "2000-01-01T00:00:03Z", - "execlog": null, - "start_time": "2000-01-01T00:00:03Z", - "user": 4 - }, - "model": "datachecking.contentchecklog", - "pk": 15 - }, - { - "fields": { - "dataset": 19, - "end_time": "2000-01-01T00:00:17Z", - "execlog": 2, - "start_time": "2000-01-01T00:00:17Z", - "user": 4 - }, - "model": "datachecking.contentchecklog", - "pk": 16 - }, - { - "fields": { - "dataset": 20, - "end_time": "2000-01-01T00:00:32Z", - "execlog": 4, - "start_time": "2000-01-01T00:00:32Z", - "user": 4 - }, - "model": "datachecking.contentchecklog", - "pk": 17 - }, - { - "fields": { - "dataset": 21, - "end_time": "2000-01-01T00:01:02Z", - "execlog": 9, - "start_time": "2000-01-01T00:01:02Z", - "user": 4 - }, - "model": "datachecking.contentchecklog", - "pk": 18 - }, - { - "fields": { - "dataset": 18, - "end_time": "2000-01-01T00:00:04Z", - "execlog": null, - "read_failed": false, - "runcomponent": 3, - "start_time": "2000-01-01T00:00:04Z", - "user": 4 - }, - "model": "datachecking.integritychecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 19, - "end_time": "2000-01-01T00:00:17Z", - "execlog": 2, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:17Z", - "user": 4 - }, - "model": "datachecking.integritychecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 20, - "end_time": "2000-01-01T00:00:32Z", - "execlog": 4, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:32Z", - "user": 4 - }, - "model": "datachecking.integritychecklog", - "pk": 3 - }, - { - "fields": { - "dataset": 18, - "end_time": "2000-01-01T00:00:36Z", - "execlog": null, - "read_failed": false, - "runcomponent": 8, - "start_time": "2000-01-01T00:00:36Z", - "user": 4 - }, - "model": "datachecking.integritychecklog", - "pk": 4 - }, - { - "fields": { - "dataset": 19, - "end_time": "2000-01-01T00:00:48Z", - "execlog": 7, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:48Z", - "user": 4 - }, - "model": "datachecking.integritychecklog", - "pk": 5 - }, - { - "fields": { - "dataset": 21, - "end_time": "2000-01-01T00:01:02Z", - "execlog": 9, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:01:02Z", - "user": 4 - }, - "model": "datachecking.integritychecklog", - "pk": 6 - } -] \ No newline at end of file diff --git a/kive/portal/fixtures/simple_run.json b/kive/portal/fixtures/simple_run.json deleted file mode 100644 index 73f262424..000000000 --- a/kive/portal/fixtures/simple_run.json +++ /dev/null @@ -1,3231 +0,0 @@ -[ - { - "fields": { - "datatype": 8, - "rule": "^[ACGTacgt]*$", - "ruletype": "regexp" - }, - "model": "metadata.basicconstraint", - "pk": 6 - }, - { - "fields": { - "datatype": 9, - "rule": "^[ACGUacgu]*$", - "ruletype": "regexp" - }, - "model": "metadata.basicconstraint", - "pk": 7 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "label", - "compounddatatype": 5, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 7 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "integer", - "compounddatatype": 5, - "datatype": 4 - }, - "model": "metadata.compounddatatypemember", - "pk": 8 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "float", - "compounddatatype": 5, - "datatype": 3 - }, - "model": "metadata.compounddatatypemember", - "pk": 9 - }, - { - "fields": { - "blankable": false, - "column_idx": 4, - "column_name": "bool", - "compounddatatype": 5, - "datatype": 2 - }, - "model": "metadata.compounddatatypemember", - "pk": 10 - }, - { - "fields": { - "blankable": false, - "column_idx": 5, - "column_name": "rna", - "compounddatatype": 5, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 11 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "label", - "compounddatatype": 6, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 12 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "label", - "compounddatatype": 7, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 13 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "PBMCseq", - "compounddatatype": 7, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 14 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "PLAseq", - "compounddatatype": 7, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 15 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "SeqToComplement", - "compounddatatype": 8, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 16 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "ComplementedSeq", - "compounddatatype": 9, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 17 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "SeqToComplement", - "compounddatatype": 10, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 18 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "ComplementedSeq", - "compounddatatype": 11, - "datatype": 9 - }, - "model": "metadata.compounddatatypemember", - "pk": 19 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "x", - "compounddatatype": 12, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 20 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "y", - "compounddatatype": 12, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 21 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "k", - "compounddatatype": 13, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 22 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "a", - "compounddatatype": 14, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 23 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "b", - "compounddatatype": 14, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 24 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "c", - "compounddatatype": 14, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 25 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "a^2", - "compounddatatype": 15, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 26 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "b^2", - "compounddatatype": 15, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 27 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "c^2", - "compounddatatype": 15, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 28 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "StrCol1", - "compounddatatype": 16, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 29 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "DNACol2", - "compounddatatype": 16, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 30 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "StrCol3", - "compounddatatype": 16, - "datatype": 1 - }, - "model": "metadata.compounddatatypemember", - "pk": 31 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "a", - "compounddatatype": 17, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 32 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "b", - "compounddatatype": 17, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 33 - }, - { - "fields": { - "blankable": false, - "column_idx": 3, - "column_name": "c", - "compounddatatype": 17, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 34 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "x", - "compounddatatype": 18, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 35 - }, - { - "fields": { - "blankable": false, - "column_idx": 2, - "column_name": "y", - "compounddatatype": 18, - "datatype": 8 - }, - "model": "metadata.compounddatatypemember", - "pk": 36 - }, - { - "fields": { - "blankable": false, - "column_idx": 1, - "column_name": "word", - "compounddatatype": 19, - "datatype": 10 - }, - "model": "metadata.compounddatatypemember", - "pk": 37 - }, - { - "fields": { - "dataset": 18, - "index": 1, - "run": 1 - }, - "model": "archive.runinput", - "pk": 1 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:16Z", - "execrecord": 2, - "reused": false, - "start_time": "2000-01-01T00:00:03Z" - }, - "model": "archive.runcomponent", - "pk": 1 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:31Z", - "execrecord": 4, - "reused": false, - "start_time": "2000-01-01T00:00:17Z" - }, - "model": "archive.runcomponent", - "pk": 2 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:04Z", - "execrecord": 1, - "reused": false, - "start_time": "2000-01-01T00:00:03Z" - }, - "model": "archive.runcomponent", - "pk": 3 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:18Z", - "execrecord": 3, - "reused": false, - "start_time": "2000-01-01T00:00:17Z" - }, - "model": "archive.runcomponent", - "pk": 4 - }, - { - "fields": { - "_redacted": null, - "_runcomponentstate": 3, - "end_time": "2000-01-01T00:00:33Z", - "execrecord": 5, - "reused": false, - "start_time": "2000-01-01T00:00:33Z" - }, - "model": "archive.runcomponent", - "pk": 5 - }, - { - "fields": { - "pipelinestep": 5, - "run": 1 - }, - "model": "archive.runstep", - "pk": 1 - }, - { - "fields": { - "pipelinestep": 6, - "run": 1 - }, - "model": "archive.runstep", - "pk": 2 - }, - { - "fields": { - "PSIC": 12, - "dest_runstep": 1 - }, - "model": "archive.runsic", - "pk": 3 - }, - { - "fields": { - "PSIC": 13, - "dest_runstep": 2 - }, - "model": "archive.runsic", - "pk": 4 - }, - { - "fields": { - "pipelineoutputcable": 14, - "run": 1 - }, - "model": "archive.runoutputcable", - "pk": 5 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:04Z", - "invoking_record": 3, - "record": 3, - "start_time": "2000-01-01T00:00:04Z" - }, - "model": "archive.execlog", - "pk": 1 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:04Z", - "invoking_record": 1, - "record": 1, - "start_time": "2000-01-01T00:00:04Z" - }, - "model": "archive.execlog", - "pk": 2 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:17Z", - "invoking_record": 4, - "record": 4, - "start_time": "2000-01-01T00:00:17Z" - }, - "model": "archive.execlog", - "pk": 3 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:18Z", - "invoking_record": 2, - "record": 2, - "start_time": "2000-01-01T00:00:18Z" - }, - "model": "archive.execlog", - "pk": 4 - }, - { - "fields": { - "end_time": "2000-01-01T00:00:33Z", - "invoking_record": 5, - "record": 5, - "start_time": "2000-01-01T00:00:33Z" - }, - "model": "archive.execlog", - "pk": 5 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step1_stderr_slurmIDJ_nodeN.txt", - "error_redacted": false, - "execlog": 2, - "install_failed": false, - "output_log": "Logs/step1_stdout_slurmIDJ_nodeN.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 1 - }, - { - "fields": { - "are_checksums_OK": true, - "code_redacted": false, - "error_log": "Logs/step2_stderr_slurmIDJ_nodeN.txt", - "error_redacted": false, - "execlog": 4, - "install_failed": false, - "output_log": "Logs/step2_stdout_slurmIDJ_nodeN.txt", - "output_redacted": false, - "return_code": 0 - }, - "model": "archive.methodoutput", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 14, - "dataset": 1, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 2, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 13, - "dataset": 3, - "num_rows": 13 - }, - "model": "librarian.datasetstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 13, - "dataset": 4, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 6, - "num_rows": 10 - }, - "model": "librarian.datasetstructure", - "pk": 5 - }, - { - "fields": { - "compounddatatype": 14, - "dataset": 7, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 6 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 8, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 7 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 9, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 8 - }, - { - "fields": { - "compounddatatype": 13, - "dataset": 10, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 9 - }, - { - "fields": { - "compounddatatype": 14, - "dataset": 13, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 10 - }, - { - "fields": { - "compounddatatype": 12, - "dataset": 14, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 11 - }, - { - "fields": { - "compounddatatype": 17, - "dataset": 15, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 12 - }, - { - "fields": { - "compounddatatype": 18, - "dataset": 16, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 13 - }, - { - "fields": { - "compounddatatype": 18, - "dataset": 17, - "num_rows": 3 - }, - "model": "librarian.datasetstructure", - "pk": 14 - }, - { - "fields": { - "compounddatatype": 19, - "dataset": 18, - "num_rows": 1 - }, - "model": "librarian.datasetstructure", - "pk": 15 - }, - { - "fields": { - "compounddatatype": 19, - "dataset": 19, - "num_rows": 1 - }, - "model": "librarian.datasetstructure", - "pk": 16 - }, - { - "fields": { - "compounddatatype": 19, - "dataset": 20, - "num_rows": 1 - }, - "model": "librarian.datasetstructure", - "pk": 17 - }, - { - "fields": { - "generator": 1 - }, - "model": "librarian.execrecord", - "pk": 1 - }, - { - "fields": { - "generator": 2 - }, - "model": "librarian.execrecord", - "pk": 2 - }, - { - "fields": { - "generator": 3 - }, - "model": "librarian.execrecord", - "pk": 3 - }, - { - "fields": { - "generator": 4 - }, - "model": "librarian.execrecord", - "pk": 4 - }, - { - "fields": { - "generator": 5 - }, - "model": "librarian.execrecord", - "pk": 5 - }, - { - "fields": { - "dataset": 18, - "execrecord": 1, - "generic_input": 26 - }, - "model": "librarian.execrecordin", - "pk": 1 - }, - { - "fields": { - "dataset": 18, - "execrecord": 2, - "generic_input": 20 - }, - "model": "librarian.execrecordin", - "pk": 2 - }, - { - "fields": { - "dataset": 19, - "execrecord": 3, - "generic_input": 21 - }, - "model": "librarian.execrecordin", - "pk": 3 - }, - { - "fields": { - "dataset": 19, - "execrecord": 4, - "generic_input": 20 - }, - "model": "librarian.execrecordin", - "pk": 4 - }, - { - "fields": { - "dataset": 20, - "execrecord": 5, - "generic_input": 21 - }, - "model": "librarian.execrecordin", - "pk": 5 - }, - { - "fields": { - "dataset": 18, - "execrecord": 1, - "generic_output": 20 - }, - "model": "librarian.execrecordout", - "pk": 1 - }, - { - "fields": { - "dataset": 19, - "execrecord": 2, - "generic_output": 21 - }, - "model": "librarian.execrecordout", - "pk": 2 - }, - { - "fields": { - "dataset": 19, - "execrecord": 3, - "generic_output": 20 - }, - "model": "librarian.execrecordout", - "pk": 3 - }, - { - "fields": { - "dataset": 20, - "execrecord": 4, - "generic_output": 21 - }, - "model": "librarian.execrecordout", - "pk": 4 - }, - { - "fields": { - "dataset": 20, - "execrecord": 5, - "generic_output": 28 - }, - "model": "librarian.execrecordout", - "pk": 5 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 4, - "step_num": 1, - "transformation": 2, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 1 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 1, - "transformation": 1, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 2 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 2, - "transformation": 4, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 3 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 5, - "step_num": 3, - "transformation": 3, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 4 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 9, - "step_num": 1, - "transformation": 6, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 5 - }, - { - "fields": { - "fill_colour": "", - "name": "", - "outputs_to_delete": [], - "pipeline": 9, - "step_num": 2, - "transformation": 6, - "x": 0.0, - "y": 0.0 - }, - "model": "pipeline.pipelinestep", - "pk": 6 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 1 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 2 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 3 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 4 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 5 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 6 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 7 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 8 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 9 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 10 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 11 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 12 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 13 - }, - { - "fields": {}, - "model": "pipeline.pipelinecable", - "pk": 14 - }, - { - "fields": { - "dest": 3, - "keep_output": false, - "pipelinestep": 1, - "source": 11, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 1 - }, - { - "fields": { - "dest": 4, - "keep_output": false, - "pipelinestep": 1, - "source": 12, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 2 - }, - { - "fields": { - "dest": 1, - "keep_output": false, - "pipelinestep": 2, - "source": 15, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 4 - }, - { - "fields": { - "dest": 11, - "keep_output": false, - "pipelinestep": 3, - "source": 13, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 5 - }, - { - "fields": { - "dest": 12, - "keep_output": false, - "pipelinestep": 3, - "source": 14, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 6 - }, - { - "fields": { - "dest": 7, - "keep_output": false, - "pipelinestep": 4, - "source": 2, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 7 - }, - { - "fields": { - "dest": 6, - "keep_output": false, - "pipelinestep": 4, - "source": 16, - "source_step": 2 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 8 - }, - { - "fields": { - "dest": 20, - "keep_output": false, - "pipelinestep": 5, - "source": 26, - "source_step": 0 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 12 - }, - { - "fields": { - "dest": 20, - "keep_output": false, - "pipelinestep": 6, - "source": 21, - "source_step": 1 - }, - "model": "pipeline.pipelinestepinputcable", - "pk": 13 - }, - { - "fields": { - "cable": 5, - "dest_pin": 21, - "source_pin": 23 - }, - "model": "pipeline.customcablewire", - "pk": 1 - }, - { - "fields": { - "cable": 5, - "dest_pin": 20, - "source_pin": 25 - }, - "model": "pipeline.customcablewire", - "pk": 2 - }, - { - "fields": { - "cable": 7, - "dest_pin": 21, - "source_pin": 20 - }, - "model": "pipeline.customcablewire", - "pk": 3 - }, - { - "fields": { - "cable": 7, - "dest_pin": 20, - "source_pin": 21 - }, - "model": "pipeline.customcablewire", - "pk": 4 - }, - { - "fields": { - "cable": 9, - "dest_pin": 21, - "source_pin": 24 - }, - "model": "pipeline.customcablewire", - "pk": 5 - }, - { - "fields": { - "cable": 9, - "dest_pin": 20, - "source_pin": 25 - }, - "model": "pipeline.customcablewire", - "pk": 6 - }, - { - "fields": { - "output_cdt": 14, - "output_idx": 1, - "output_name": "D1_out", - "pipeline": 4, - "source": 5, - "source_step": 1 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 3 - }, - { - "fields": { - "output_cdt": 12, - "output_idx": 1, - "output_name": "E1_out", - "pipeline": 5, - "source": 16, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 9 - }, - { - "fields": { - "output_cdt": 13, - "output_idx": 2, - "output_name": "E2_out", - "pipeline": 5, - "source": 8, - "source_step": 3 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 10 - }, - { - "fields": { - "output_cdt": null, - "output_idx": 3, - "output_name": "E3_rawout", - "pipeline": 5, - "source": 10, - "source_step": 3 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 11 - }, - { - "fields": { - "output_cdt": 19, - "output_idx": 1, - "output_name": "basic_out", - "pipeline": 9, - "source": 21, - "source_step": 2 - }, - "model": "pipeline.pipelineoutputcable", - "pk": 14 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 1 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 2 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 3 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 4 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 5 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 6 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 7 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 8 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 9 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 10 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 11 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 12 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 13 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 14 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 15 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 16 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 17 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 18 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 19 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 20 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 21 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 22 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 23 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 24 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 25 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 26 - }, - { - "fields": { - "x": 0.0, - "y": 0.0 - }, - "model": "transformation.transformationxput", - "pk": 28 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 2 - }, - "model": "transformation.xputstructure", - "pk": 1 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 3 - }, - "model": "transformation.xputstructure", - "pk": 2 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 4 - }, - "model": "transformation.xputstructure", - "pk": 3 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": 5, - "min_row": null, - "transf_xput": 5 - }, - "model": "transformation.xputstructure", - "pk": 4 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": null, - "min_row": null, - "transf_xput": 6 - }, - "model": "transformation.xputstructure", - "pk": 5 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 7 - }, - "model": "transformation.xputstructure", - "pk": 6 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 8 - }, - "model": "transformation.xputstructure", - "pk": 7 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": null, - "min_row": null, - "transf_xput": 11 - }, - "model": "transformation.xputstructure", - "pk": 8 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 12 - }, - "model": "transformation.xputstructure", - "pk": 9 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": null, - "min_row": null, - "transf_xput": 13 - }, - "model": "transformation.xputstructure", - "pk": 10 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": 10, - "transf_xput": 14 - }, - "model": "transformation.xputstructure", - "pk": 11 - }, - { - "fields": { - "compounddatatype": 14, - "max_row": 5, - "min_row": null, - "transf_xput": 16 - }, - "model": "transformation.xputstructure", - "pk": 12 - }, - { - "fields": { - "compounddatatype": 12, - "max_row": 5, - "min_row": null, - "transf_xput": 17 - }, - "model": "transformation.xputstructure", - "pk": 13 - }, - { - "fields": { - "compounddatatype": 13, - "max_row": null, - "min_row": null, - "transf_xput": 18 - }, - "model": "transformation.xputstructure", - "pk": 14 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 20 - }, - "model": "transformation.xputstructure", - "pk": 15 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 21 - }, - "model": "transformation.xputstructure", - "pk": 16 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 22 - }, - "model": "transformation.xputstructure", - "pk": 17 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 23 - }, - "model": "transformation.xputstructure", - "pk": 18 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 26 - }, - "model": "transformation.xputstructure", - "pk": 19 - }, - { - "fields": { - "compounddatatype": 19, - "max_row": null, - "min_row": null, - "transf_xput": 28 - }, - "model": "transformation.xputstructure", - "pk": 21 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "A1_rawin", - "transformation": 1 - }, - "model": "transformation.transformationinput", - "pk": 1 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "B1_in", - "transformation": 2 - }, - "model": "transformation.transformationinput", - "pk": 3 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "B2_in", - "transformation": 2 - }, - "model": "transformation.transformationinput", - "pk": 4 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "C1_in", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 6 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "C2_in", - "transformation": 3 - }, - "model": "transformation.transformationinput", - "pk": 7 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "D1_in", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 11 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "D2_in", - "transformation": 4 - }, - "model": "transformation.transformationinput", - "pk": 12 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "E1_in", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 13 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "E2_in", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 14 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "E3_rawin", - "transformation": 5 - }, - "model": "transformation.transformationinput", - "pk": 15 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "strings", - "transformation": 6 - }, - "model": "transformation.transformationinput", - "pk": 20 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "strings", - "transformation": 7 - }, - "model": "transformation.transformationinput", - "pk": 22 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "raw", - "transformation": 8 - }, - "model": "transformation.transformationinput", - "pk": 24 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "basic_in", - "transformation": 9 - }, - "model": "transformation.transformationinput", - "pk": 26 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "A1_out", - "transformation": 1 - }, - "model": "transformation.transformationoutput", - "pk": 2 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "B1_out", - "transformation": 2 - }, - "model": "transformation.transformationoutput", - "pk": 5 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "C1_out", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 8 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "C2_rawout", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 9 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "C3_rawout", - "transformation": 3 - }, - "model": "transformation.transformationoutput", - "pk": 10 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "D1_out", - "transformation": 4 - }, - "model": "transformation.transformationoutput", - "pk": 16 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "E1_out", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 17 - }, - { - "fields": { - "dataset_idx": 2, - "dataset_name": "E2_out", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 18 - }, - { - "fields": { - "dataset_idx": 3, - "dataset_name": "E3_rawout", - "transformation": 5 - }, - "model": "transformation.transformationoutput", - "pk": 19 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "same_strings", - "transformation": 6 - }, - "model": "transformation.transformationoutput", - "pk": 21 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "untouched_strings", - "transformation": 7 - }, - "model": "transformation.transformationoutput", - "pk": 23 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "same_raw", - "transformation": 8 - }, - "model": "transformation.transformationoutput", - "pk": 25 - }, - { - "fields": { - "dataset_idx": 1, - "dataset_name": "basic_out", - "transformation": 9 - }, - "model": "transformation.transformationoutput", - "pk": 28 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:00Z", - "email": "lennon@thebeatles.com", - "first_name": "", - "groups": [ - 1 - ], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$ZhSUgin9etTx$iu/UkHk8iOp07SKy+n9gXsizYuSC6w4GBcdjEyhw9wU=", - "user_permissions": [], - "username": "john" - }, - "model": "auth.user", - "pk": 2 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:00Z", - "email": "starr@thebeatles.com", - "first_name": "", - "groups": [], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$xwdQ7NELLn7b$xpntFu+LifV9e6IfM3HWdvSZMnng2NZRBdr3XI6VAHM=", - "user_permissions": [], - "username": "ringo" - }, - "model": "auth.user", - "pk": 3 - }, - { - "fields": { - "date_joined": "2000-01-01T00:00:02Z", - "email": "bob@talabs.com", - "first_name": "", - "groups": [ - 1 - ], - "is_active": true, - "is_staff": false, - "is_superuser": false, - "last_login": null, - "last_name": "", - "password": "pbkdf2_sha256$24000$8FNDXIoI9a92$WHVmogFUid/TskNTsEvj5z1uofqTnupBZE/hJ1qJMTE=", - "user_permissions": [], - "username": "bob" - }, - "model": "auth.user", - "pk": 4 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:00Z", - "description": "String consisting of ACGTacgt", - "groups_allowed": [ - 1 - ], - "name": "DNANucSeq", - "prototype": null, - "restricts": [ - 1 - ], - "user": 2, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 8 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:00Z", - "description": "String consisting of ACGUacgu", - "groups_allowed": [ - 1 - ], - "name": "RNANucSeq", - "prototype": null, - "restricts": [ - 1 - ], - "user": 2, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 9 - }, - { - "fields": { - "date_created": "2000-01-01T00:00:02Z", - "description": "sequences of ASCII characters", - "groups_allowed": [ - 1 - ], - "name": "my_string", - "prototype": null, - "restricts": [ - 1 - ], - "user": 4, - "users_allowed": [] - }, - "model": "metadata.datatype", - "pk": 10 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [], - "name": "", - "user": 2, - "users_allowed": [ - 3 - ] - }, - "model": "metadata.compounddatatype", - "pk": 6 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 7 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 8 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 9 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 10 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 11 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 12 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 13 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 14 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 15 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 16 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 17 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 2, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 18 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "name": "", - "user": 4, - "users_allowed": [] - }, - "model": "metadata.compounddatatype", - "pk": 19 - }, - { - "fields": { - "_runstate": 3, - "description": "", - "end_time": "2000-01-01T00:00:34Z", - "groups_allowed": [ - 1 - ], - "name": "", - "parent_runstep": null, - "paused_by": null, - "pipeline": 9, - "priority": 0, - "purged": false, - "runbatch": null, - "sandbox_path": "/home/rliang/Documents/Kive/media/Sandboxes/userbob_run1_56ltyJ", - "start_time": "2000-01-01T00:00:03Z", - "stopped_by": null, - "time_queued": "2017-04-25T00:53:05.656Z", - "user": 4, - "users_allowed": [] - }, - "model": "archive.run", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "ab7afe2f453d27fd168641a1f9271f26", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_triplet.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:53:03.562Z", - "name": "triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "02d5b66858d003b81ae3dc712b74e85c", - "_redacted": false, - "dataset_file": "Datasets/2017_04/doublet_cdt.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:53:03.632Z", - "name": "doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "29feb269edcac613d6eea432c14640bd", - "_redacted": false, - "dataset_file": "Datasets/2017_04/singlet_cdt_large.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:53:03.697Z", - "name": "singlet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 3 - }, - { - "fields": { - "MD5_checksum": "00fd50bfe7ac63e2165a442f4101e05c", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_singlet.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:53:03.751Z", - "name": "singlet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 4 - }, - { - "fields": { - "MD5_checksum": "7dc85e11b5c02e434af5bd3b3da9938e", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_raw.fasta", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:53:03.823Z", - "name": "raw_DS", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 5 - }, - { - "fields": { - "MD5_checksum": "542676b23e121d16db8d41ccdae65fd1", - "_redacted": false, - "dataset_file": "", - "date_created": "2000-01-01T00:00:01Z", - "description": "", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:53:03.848Z", - "name": "D1_in_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 6 - }, - { - "fields": { - "MD5_checksum": "56f9e5585b9d699829e2fda931d944f8", - "_redacted": false, - "dataset_file": "Datasets/2017_04/C1_in_triplet.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "triplet 3 rows", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:53:03.904Z", - "name": "C1_in_triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 7 - }, - { - "fields": { - "MD5_checksum": "10a4bd15d1300fcaf5a3bdad54ef1d28", - "_redacted": false, - "dataset_file": "", - "date_created": "2000-01-01T00:00:01Z", - "description": "", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:53:03.971Z", - "name": "C2_in_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 8 - }, - { - "fields": { - "MD5_checksum": "10a4bd15d1300fcaf5a3bdad54ef1d28", - "_redacted": false, - "dataset_file": "Datasets/2017_04/E11_32_output.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "result of E11_32 fed by doublet_cdt.csv", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:53:04.020Z", - "name": "E11_32 output doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 9 - }, - { - "fields": { - "MD5_checksum": "00fd50bfe7ac63e2165a442f4101e05c", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_singlet_PL7q4Af.csv", - "date_created": "2000-01-01T00:00:01Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:53:04.079Z", - "name": "raw", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 10 - }, - { - "fields": { - "MD5_checksum": "7dc85e11b5c02e434af5bd3b3da9938e", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_raw_NlpJJgR.fasta", - "date_created": "2000-01-01T00:00:02Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:53:04.120Z", - "name": "C2_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 11 - }, - { - "fields": { - "MD5_checksum": "7dc85e11b5c02e434af5bd3b3da9938e", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_raw_6uw8GVN.fasta", - "date_created": "2000-01-01T00:00:02Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:53:04.152Z", - "name": "C3_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 12 - }, - { - "fields": { - "MD5_checksum": "56f9e5585b9d699829e2fda931d944f8", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step_0_triplet_3_rows.csv", - "date_created": "2000-01-01T00:00:02Z", - "description": "lol", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:53:04.187Z", - "name": "triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 13 - }, - { - "fields": { - "MD5_checksum": "83957f8aea1c75da9f7fbf9bc90da979", - "_redacted": false, - "dataset_file": "Datasets/2017_04/doublet_remuxed_from_t3r.csv", - "date_created": "2000-01-01T00:00:02Z", - "description": "doublet remuxed from triplet", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:53:04.254Z", - "name": "E1_out", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 14 - }, - { - "fields": { - "MD5_checksum": "d83866e4c23cd9503aa2f9fc565502b5", - "_redacted": false, - "dataset_file": "Datasets/2017_04/DNA_triplet.csv", - "date_created": "2000-01-01T00:00:02Z", - "description": "DNA triplet data", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:53:04.313Z", - "name": "DNA_triplet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 15 - }, - { - "fields": { - "MD5_checksum": "9eab0dfed5fcc89bd581f61995816ef0", - "_redacted": false, - "dataset_file": "Datasets/2017_04/E01_21_DNA_doublet.csv", - "date_created": "2000-01-01T00:00:02Z", - "description": "DNA doublet data coming from DNA_triplet.csv but remultiplexed according to cable E01_21", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:53:04.471Z", - "name": "E01_21_DNA_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 16 - }, - { - "fields": { - "MD5_checksum": "9b55b9431da6dba74ea61edf8e2ac044", - "_redacted": false, - "dataset_file": "Datasets/2017_04/E21_41_DNA_doublet.csv", - "date_created": "2000-01-01T00:00:02Z", - "description": "DNA doublet data coming from DNA_triplet.csv but remultiplexed according to cable E21_41", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:53:04.572Z", - "name": "E21_41_DNA_doublet", - "user": 2, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 17 - }, - { - "fields": { - "MD5_checksum": "092ef90fe0201f4b002cd200f7754d45", - "_redacted": false, - "dataset_file": "Datasets/2017_04/tmpWmKEBj", - "date_created": "2000-01-01T00:00:03Z", - "description": "blahblahblah", - "external_path": "", - "externalfiledirectory": null, - "file_source": null, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:53:05.593Z", - "name": "blahblah", - "user": 4, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 18 - }, - { - "fields": { - "MD5_checksum": "092ef90fe0201f4b002cd200f7754d45", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step1_same_strings.csv", - "date_created": "2000-01-01T00:00:15Z", - "description": "Generated data from a run of pipeline \"P_basic:1 (v1)\" started at 2017-04-25 00:53:05.785593+00:00 by bob\nrun: 1\nuser: bob\nstep: 1\noutput: same_strings", - "external_path": "", - "externalfiledirectory": null, - "file_source": 1, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:53:17.548Z", - "name": "run1_step1_outputsame_strings", - "user": 4, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 19 - }, - { - "fields": { - "MD5_checksum": "092ef90fe0201f4b002cd200f7754d45", - "_redacted": false, - "dataset_file": "Datasets/2017_04/step2_same_strings.csv", - "date_created": "2000-01-01T00:00:30Z", - "description": "Generated data from a run of pipeline \"P_basic:1 (v1)\" started at 2017-04-25 00:53:05.785593+00:00 by bob\nrun: 1\nuser: bob\nstep: 2\noutput: same_strings", - "external_path": "", - "externalfiledirectory": null, - "file_source": 2, - "groups_allowed": [ - 1 - ], - "last_time_checked": "2017-04-25T00:53:32.864Z", - "name": "run1_step2_outputsame_strings", - "user": 4, - "users_allowed": [] - }, - "model": "librarian.dataset", - "pk": 20 - }, - { - "fields": { - "description": "Just a CR", - "filename": "generic_script.py", - "groups_allowed": [ - 1 - ], - "name": "genericCR", - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 1 - }, - { - "fields": { - "description": "a script to do nothing", - "filename": "noop.sh", - "groups_allowed": [ - 1 - ], - "name": "noop", - "user": 4, - "users_allowed": [] - }, - "model": "method.coderesource", - "pk": 2 - }, - { - "fields": { - "MD5_checksum": "e353a62fe8cb6a56d31b8c65d70b217d", - "coderesource": 1, - "content_file": "CodeResources/generic_script.py", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "desc", - "revision_name": "v1", - "revision_number": 1, - "revision_parent": null, - "user": 2, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 1 - }, - { - "fields": { - "MD5_checksum": "d218c00725d473639f347d456de381d8", - "coderesource": 2, - "content_file": "CodeResources/fdopen", - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "1", - "revision_number": 1, - "revision_parent": null, - "user": 4, - "users_allowed": [] - }, - "model": "method.coderesourcerevision", - "pk": 2 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 1 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 2, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 2 - }, - { - "fields": { - "driver": 1, - "family": 1, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 3, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 3 - }, - { - "fields": { - "driver": 2, - "family": 2, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 6 - }, - { - "fields": { - "driver": 2, - "family": 3, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 7 - }, - { - "fields": { - "driver": 2, - "family": 4, - "groups_allowed": [ - 1 - ], - "reusable": 1, - "revision_number": 1, - "revision_parent": null, - "tainted": false, - "threads": 1, - "users_allowed": [] - }, - "model": "method.method", - "pk": 8 - }, - { - "fields": { - "description": "Holds methods A/B/C", - "groups_allowed": [ - 1 - ], - "name": "method_family", - "user": 2, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 1 - }, - { - "fields": { - "description": "a method to do nothing to strings", - "groups_allowed": [ - 1 - ], - "name": "string noop", - "user": 4, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 2 - }, - { - "fields": { - "description": "a TOTALLY DIFFERENT method that TOTALLY does SOMETHING to strings by leaving them alone", - "groups_allowed": [ - 1 - ], - "name": "string trivial", - "user": 4, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 3 - }, - { - "fields": { - "description": "do nothing to raw data", - "groups_allowed": [ - 1 - ], - "name": "raw noop", - "user": 4, - "users_allowed": [] - }, - "model": "method.methodfamily", - "pk": 4 - }, - { - "fields": { - "description": "PF desc", - "groups_allowed": [ - 1 - ], - "name": "Pipeline_family", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 1 - }, - { - "fields": { - "description": "Innermost pipeline", - "groups_allowed": [ - 1 - ], - "name": "P_basic", - "user": 2, - "users_allowed": [] - }, - "model": "pipeline.pipelinefamily", - "pk": 2 - }, - { - "fields": { - "family": 1, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 4 - }, - { - "fields": { - "family": 1, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 2, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 5 - }, - { - "fields": { - "family": 2, - "groups_allowed": [ - 1 - ], - "published": false, - "revision_number": 1, - "revision_parent": null, - "users_allowed": [] - }, - "model": "pipeline.pipeline", - "pk": 9 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "A_desc", - "revision_name": "mA_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 1 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "B_desc", - "revision_name": "mB_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 2 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "C_desc", - "revision_name": "mC_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 3 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "D", - "revision_name": "pD_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 4 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:00Z", - "revision_desc": "E", - "revision_name": "pE_name", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 5 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 4, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 6 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 4, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 7 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:02Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 4, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 8 - }, - { - "fields": { - "groups_allowed": [ - 1 - ], - "revision_DateTime": "2000-01-01T00:00:03Z", - "revision_desc": "first version", - "revision_name": "v1", - "user": 2, - "users_allowed": [] - }, - "model": "transformation.transformation", - "pk": 9 - }, - { - "fields": { - "dataset": 1, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 2, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 3, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 3 - }, - { - "fields": { - "dataset": 4, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 4 - }, - { - "fields": { - "dataset": 6, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 5 - }, - { - "fields": { - "dataset": 7, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 6 - }, - { - "fields": { - "dataset": 8, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 7 - }, - { - "fields": { - "dataset": 9, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 8 - }, - { - "fields": { - "dataset": 10, - "end_time": "2000-01-01T00:00:01Z", - "execlog": null, - "start_time": "2000-01-01T00:00:01Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 9 - }, - { - "fields": { - "dataset": 13, - "end_time": "2000-01-01T00:00:02Z", - "execlog": null, - "start_time": "2000-01-01T00:00:02Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 10 - }, - { - "fields": { - "dataset": 14, - "end_time": "2000-01-01T00:00:02Z", - "execlog": null, - "start_time": "2000-01-01T00:00:02Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 11 - }, - { - "fields": { - "dataset": 15, - "end_time": "2000-01-01T00:00:02Z", - "execlog": null, - "start_time": "2000-01-01T00:00:02Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 12 - }, - { - "fields": { - "dataset": 16, - "end_time": "2000-01-01T00:00:02Z", - "execlog": null, - "start_time": "2000-01-01T00:00:02Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 13 - }, - { - "fields": { - "dataset": 17, - "end_time": "2000-01-01T00:00:02Z", - "execlog": null, - "start_time": "2000-01-01T00:00:02Z", - "user": 2 - }, - "model": "datachecking.contentchecklog", - "pk": 14 - }, - { - "fields": { - "dataset": 18, - "end_time": "2000-01-01T00:00:03Z", - "execlog": null, - "start_time": "2000-01-01T00:00:03Z", - "user": 4 - }, - "model": "datachecking.contentchecklog", - "pk": 15 - }, - { - "fields": { - "dataset": 19, - "end_time": "2000-01-01T00:00:16Z", - "execlog": 2, - "start_time": "2000-01-01T00:00:16Z", - "user": 4 - }, - "model": "datachecking.contentchecklog", - "pk": 16 - }, - { - "fields": { - "dataset": 20, - "end_time": "2000-01-01T00:00:31Z", - "execlog": 4, - "start_time": "2000-01-01T00:00:31Z", - "user": 4 - }, - "model": "datachecking.contentchecklog", - "pk": 17 - }, - { - "fields": { - "dataset": 18, - "end_time": "2000-01-01T00:00:03Z", - "execlog": null, - "read_failed": false, - "runcomponent": 3, - "start_time": "2000-01-01T00:00:03Z", - "user": 4 - }, - "model": "datachecking.integritychecklog", - "pk": 1 - }, - { - "fields": { - "dataset": 19, - "end_time": "2000-01-01T00:00:16Z", - "execlog": 2, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:16Z", - "user": 4 - }, - "model": "datachecking.integritychecklog", - "pk": 2 - }, - { - "fields": { - "dataset": 20, - "end_time": "2000-01-01T00:00:31Z", - "execlog": 4, - "read_failed": false, - "runcomponent": null, - "start_time": "2000-01-01T00:00:31Z", - "user": 4 - }, - "model": "datachecking.integritychecklog", - "pk": 3 - } -] \ No newline at end of file diff --git a/kive/portal/management/commands/update_test_fixtures.py b/kive/portal/management/commands/update_test_fixtures.py index 1b7b00ee7..7c65795ce 100755 --- a/kive/portal/management/commands/update_test_fixtures.py +++ b/kive/portal/management/commands/update_test_fixtures.py @@ -1,16 +1,13 @@ from __future__ import print_function from datetime import datetime import json -from operator import itemgetter import os import shutil import sys from django.contrib.auth.models import User -from django.core.files.base import ContentFile from django.core.management import call_command from django.core.management.base import BaseCommand -from django.db import transaction from django.utils.dateparse import parse_datetime from django.utils.timezone import utc from django.conf import settings @@ -19,14 +16,6 @@ from file_access_utils import compute_md5 import kive.testing_utils as tools from librarian.models import Dataset -from metadata.models import CompoundDatatype, everyone_group -from method.models import CodeResource, MethodFamily, Method,\ - CodeResourceRevision -import pipeline.models -from pipeline.models import PipelineFamily -import sandbox.tests -from fleet.workers import Manager -from fleet.tests_slurmscheduler import execute_simple_run, execute_nested_run from portal.management.commands.reset import Command as ResetCommand @@ -232,142 +221,6 @@ def create_step(self, pipeline, method, input_source): self.next_step_num += 1 return step - def create_method_from_string(self, name, source, user, - input_names, output_names, - dep_lst=(), init_code_resource=None): - """ Create a method and code_resource_revision from source code held in a string. - - @param name: the name of the method as it will appear in Kive. - @param source: source code held in a string - @param input_names: list of strings to name raw inputs - @param output_names: list of strings to name raw outputs - @param dep_lst: a list of tuples: (code_res_rev, pathstr, filestr) - defining the set of code resources this method depends on. - @param init_code_resource: the code resource of the __init__.py code that - will be copied into each package directory. - @return: a new Method object that has been saved - - """ - if (init_code_resource is None) and len(dep_lst) > 0: - raise RuntimeError("non-empty dependency list requires an __init__ code resource.") - with transaction.atomic(): - code_res_rev = self.create_code_resource(name, name+".py", source, user) - return self.create_method_from_resource(name, code_res_rev, user, - input_names, output_names, - dep_lst, init_code_resource) - - def create_method_from_resource(self, methname, code_res_rev, - user, input_names, output_names, dep_lst, - init_code_resource): - """ Create a method from a CodeResourceRevision - - @param name: the name of the method as it will appear in Kive - @param fname: file name containing source code - @param input_names: list of strings to name raw inputs - @param output_names: list of strings to name raw outputs - @param dep_lst: a list of code resources on which this code is dependent. - The list contains a 3-tuple: (resource, path (string), filename (string)) - The path consists of an os.sep ('/') separated string. - @param init_code_resource: the code resource of the __init__.py code that - will be copied into each package directory. This argument may be None. - @return: a new Method object that has been saved - - NOTE: also see create_method_file(), which reads a source string from - a specified file. - """ - with transaction.atomic(): - family = MethodFamily(name=methname, user=user) - family.save() - family.clean() - family.grant_everyone_access() - method = family.members.create(revision_name='first', - driver=code_res_rev, - user=user) - method.save() - for i, input_name in enumerate(input_names, 1): - method.inputs.create(dataset_name=input_name, dataset_idx=i) - for i, output_name in enumerate(output_names, 1): - method.outputs.create(dataset_name=output_name, dataset_idx=i) - for dep_resource, p, f in dep_lst: - method.dependencies.create(method=method, - requirement=dep_resource, - path=p, filename=f) - # must now determine all subdirectories (package names) used by the dependencies - # of this method, so as to install __init__.py files into them - if init_code_resource is not None: - for packname in self.fillpathset(frozenset([k[1] for k in dep_lst])): - method.dependencies.create(method=method, - requirement=init_code_resource, - path=packname, - filename="__init__.py") - method.clean() - method.grant_everyone_access() - return method - - def create_code_resource(self, name, filename, source, user): - """ Create a new code resource. - @param name: the name of the new code resource - @param filename: the filename of the resource - @param source: source code held in a string - @param user: the user - - @return: a new CodeResourceRevision object that has been saved - """ - with transaction.atomic(): - resource = CodeResource(name=name, filename=filename, user=user) - resource.save() - resource.clean() - resource.grant_everyone_access() - - revision = CodeResourceRevision(coderesource=resource, user=user) - revision.content_file.save(filename, ContentFile(source), save=False) - revision.clean() # calculates md5 - revision.save() - revision.grant_everyone_access() - resource.clean() - return revision - - def read_dep_resource(self, user, rootdir, resname): - """ Create a new code resource from file, returning it as a - CodeResourceRevision instance. - - @param user: the current user - @param rootdir: the name of the root directory in which to search for the source - code file. - @param resname: the name of the python module to read in. - @returns: (code_res_rev, pathname, filename), where pathname and filename are - strings. - NOTE: the resname may be a qualified import (E.g. a.b.c.py), using periods - to denote module boundary names. - In this case, this routine will attempt to load the file 'rootdir/a/b/c.py', - and the pathname and filename returned would be 'a/b' and 'c.py', respectively. - """ - # determine the filename of the file in the sandbox - nlst = [s.strip() for s in resname.split(".")] - if len(nlst) < 2: - raise RuntimeError("resname is too short: " + resname) - # remove the last two elements of the list, concatenate them with an intermediate period - # and call this new string the filename. - ext = nlst.pop() - fname = "%s.%s" % (nlst.pop(), ext) - sandbox_path = os.path.join(*nlst) - # the name of the file to read to get the source code - readfilename = os.path.join(rootdir, sandbox_path, fname) - with open(readfilename, "r") as f: - srcstr = f.read() - - # NOTE: we create resources with a file name that is INDEPENDENT of the directory name. - # So, we do NOT use sandbox_path in creating the code resource, - # i.e. resource names use the file name only. - # This means that we may not have duplicate file names, even if they normally reside - # in different sub-directories. - # However, we do pass the sandbox_path back up from this routine, because when - # we later create method dependencies, we have to know where in the file system the - # imported code dependency has to reside at run-time. - res = self.create_code_resource(fname, fname, srcstr, user) - - return (res, sandbox_path, fname) - def set_position(self, objlst): """Set the x, y screen coordinates of the objects in the list along a diagonal line (top left to bottom right) @@ -378,871 +231,6 @@ def set_position(self, objlst): obj.save() -class EMSandboxTestEnvironmentBuilder(FixtureBuilder): - def get_name(self): - return 'em_sandbox_test_environment.json' - - def build(self): - tools.create_eric_martin_test_environment(self) - tools.create_sandbox_testing_tools_environment(self) - - pE_run = self.pE.pipeline_instances.create(user=self.myUser, name='pE_run') - pE_run.grant_everyone_access() - pE_run.inputs.create(dataset=self.triplet_dataset, index=1) - pE_run.inputs.create(dataset=self.singlet_dataset, index=2) - pE_run.inputs.create(dataset=self.raw_dataset, index=3) - - -class ArchiveTestEnvironmentBuilder(FixtureBuilder): - def get_name(self): - return 'archive_test_environment.json' - - def build(self): - tools.create_archive_test_environment(self) - - -class ArchiveNoRunsTestEnvironmentBuilder(FixtureBuilder): - def get_name(self): - return 'archive_no_runs_test_environment.json' - - def build(self): - tools.create_archive_no_runs_test_environment(self) - - -class SimpleRunBuilder(FixtureBuilder): - def get_name(self): - return 'simple_run.json' - - def build(self): - execute_simple_run(self) - - -class DeepNestedRunBuilder(FixtureBuilder): - def get_name(self): - return 'deep_nested_run.json' - - def build(self): - execute_nested_run(self) - - -class RestoreReusableDatasetBuilder(FixtureBuilder): - def get_name(self): - return 'restore_reusable_dataset.json' - - def build(self): - user = User.objects.first() - pipeline1, _pipeline2 = self.create_pipelines(user) - - DATASET_CONTENT = """\ -x,y -0,1 -2,3 -4,5 -6,7 -8,9 -""" - dataset_file = ContentFile(DATASET_CONTENT) - dataset = Dataset(user=user, - name="pairs", - MD5_checksum=compute_md5(dataset_file)) - dataset.dataset_file.save(name='pairs.csv', content=dataset_file) - dataset.save() - dataset.clean() - dataset.grant_everyone_access() - - run = Manager.execute_pipeline(user=user, pipeline=pipeline1, inputs=[dataset]).get_last_run() - run.collect_garbage() # Delete sandbox directories - - def create_pipelines(self, user): - """ Create two pipelines: sums_only and sums_and_products. - - @return: (pipeline1, pipeline2) - """ - SHUFFLED_SUMS_AND_PRODUCTS_SOURCE = """\ -#! /usr/bin/env python - -from argparse import FileType, ArgumentParser -import csv -import os -from random import shuffle - -parser = ArgumentParser( - description="Takes CSV with (x,y), outputs CSV with (x+y),(x*y)"); -parser.add_argument("input_csv", - type=FileType('rU'), - help="CSV containing (x,y) pairs"); -parser.add_argument("output_csv", - type=FileType('wb'), - help="CSV containing (x+y,xy) pairs"); -args = parser.parse_args(); - -reader = csv.DictReader(args.input_csv); -writer = csv.DictWriter(args.output_csv, - ['sum', 'product'], - lineterminator=os.linesep) -writer.writeheader() - -rows = list(reader) -shuffle(rows) # Makes this version reusable, but not deterministic -for row in rows: - x = int(row['x']) - y = int(row['y']) - writer.writerow(dict(sum=x+y, product=x*y)) -""" - TOTAL_SOURCE_TEMPLATE = """\ -#!/usr/bin/env python - -from argparse import FileType, ArgumentParser -import csv -from operator import itemgetter -import os - -parser = ArgumentParser(description='Calculate the total of a column.'); -parser.add_argument("input_csv", - type=FileType('rU'), - help="CSV containing (sum,product) pairs"); -parser.add_argument("output_csv", - type=FileType('wb'), - help="CSV containing one (sum,product) pair"); -args = parser.parse_args(); - -reader = csv.DictReader(args.input_csv); -writer = csv.DictWriter(args.output_csv, - ['sum', 'product'], - lineterminator=os.linesep) -writer.writeheader() - -# Copy first row unchanged -for row in reader: - writer.writerow(row) - break - -sum_total = 0 -product_total = 0 -writer.writerow(dict(sum=sum_total, product=product_total)) -""" - total_sums_source = TOTAL_SOURCE_TEMPLATE.replace( - "sum_total = 0", - "sum_total = sum(map(int, map(itemgetter('sum'), reader)))") - total_products_source = TOTAL_SOURCE_TEMPLATE.replace( - "product_total = 0", - "product_total = sum(map(int, map(itemgetter('product'), reader)))") - - sums_and_products = self.create_method_from_string( - 'sums_and_products', - SHUFFLED_SUMS_AND_PRODUCTS_SOURCE, - user, - ['pairs'], - ['sums_and_products']) - sums_and_products.reusable = Method.REUSABLE - sums_and_products.save() - sums_and_products.clean() - total_sums = self.create_method_from_string('total_sums', - total_sums_source, - user, - ['sums_and_products'], - ['total_sums']) - total_products = self.create_method_from_string('total_products', - total_products_source, - user, - ['sums_and_products'], - ['total_products']) - with transaction.atomic(): - family = PipelineFamily(name='sums and products', user=user) - family.save() - family.clean() - family.grant_everyone_access() - - pipeline1 = family.members.create(revision_name='sums only', - user=user) - pipeline1.clean() - pipeline1.grant_everyone_access() - - self.next_step_num = 1 - self.next_output_num = 1 - input1 = pipeline1.inputs.create(dataset_name='pairs', - dataset_idx=1) - step1_1 = self.create_step(pipeline1, sums_and_products, input1) - step1_1.outputs_to_delete.add(sums_and_products.outputs.first()) - step1_2 = self.create_step(pipeline1, total_sums, step1_1) - self.create_cable(step1_2, pipeline1) - pipeline1.create_outputs() - self.set_position([input1, - step1_1, - step1_2, - pipeline1.outputs.first()]) - pipeline1.complete_clean() - - pipeline2 = family.members.create(revision_name='sums and products', - revision_parent=pipeline1, - user=user) - pipeline2.clean() - pipeline2.grant_everyone_access() - self.next_step_num = 1 - self.next_output_num = 1 - input2 = pipeline2.inputs.create(dataset_name='pairs', - dataset_idx=1) - step2_1 = self.create_step(pipeline2, sums_and_products, input2) - step2_1.outputs_to_delete.add(sums_and_products.outputs.first()) - step2_2 = self.create_step(pipeline2, total_sums, step2_1) - step2_3 = self.create_step(pipeline2, total_products, step2_1) - self.create_cable(step2_2, pipeline2) - self.create_cable(step2_3, pipeline2) - pipeline2.create_outputs() - self.set_position([input2, - step2_1, - step2_2, - pipeline2.outputs.first(), - step2_3, - pipeline2.outputs.last()]) - pipeline2.complete_clean() - return pipeline1, pipeline2 - - -class DemoBuilder(FixtureBuilder): - """This demo includes two pipelines: - a) sums and products - b) MiCallDemo - """ - - def __init__(self): - # source code for the MiCallDemo is copied from this directory - self.src_dir = os.path.join("..", "samplecode", "MiCallDemo") - - def get_name(self): - return 'demo.json' - - def create_internal_cable(self, srcobj, src_outpnum, dststep, dst_inpnum): - """Create a cable between a source's output and a destination step's input. - srcobj: either PipelineStep OR a pipeline's TransformationInput - object (the pipeline's input) - - dstobj: a PipelineStep - - NOTE: the source and destination numbers are 1-based. - """ - try: - # see if we can get the outputs of the src - # srclst=srcobj.transformation.sorted_outputs - srclst = srcobj.outputs - src_output = srclst[src_outpnum-1] - src_stepnum = srcobj.step_num - except AttributeError: - # we are connecting from a pipeline input - # we need a TranformationInput object of this pipeline - src_output = srcobj - src_stepnum = 0 - - try: - # are we connecting to a step? - dstlst = dststep.transformation.sorted_inputs - dst_input = dstlst[dst_inpnum-1] - cable = dststep.cables_in.create(source=src_output, - source_step=src_stepnum, - dest=dst_input) - except AttributeError: - print("Connecting to a destination step failed ") - raise - - return cable - - def create_PL_output(self, pipeline, stepobj, step_outpnum, ploutname, ploutnum): - """Create a pipeline output cable for a step's output. - NOTE: ploutname must be unique within a pipeline. - """ - srclst = stepobj.outputs - src_output = srclst[step_outpnum-1] - stepnum = stepobj.step_num - return pipeline.create_raw_outcable(ploutname, ploutnum, - stepnum, src_output) - - def build(self): - """Create and run two pipelines: - a) Sums and Products - b) MiCall Demo - """ - self.build_SP() - self.build_MC() - - def build_SP(self): - user = User.objects.first() - pipeline1 = self.create_SP_pipelines(user) - - DATASET_CONTENT = """\ -x,y -0,1 -2,3 -4,5 -6,7 -8,9 -""" - dataset_file = ContentFile(DATASET_CONTENT) - dataset = Dataset(user=user, - name="pairs", - MD5_checksum=compute_md5(dataset_file)) - dataset.dataset_file.save(name='pairs.csv', content=dataset_file) - dataset.save() - dataset.clean() - dataset.grant_everyone_access() - - run = Manager.execute_pipeline(user=user, pipeline=pipeline1, inputs=[dataset]).get_last_run() - run.collect_garbage() # Delete sandbox directories - - def create_SP_pipelines(self, user): - """ Create a pipeline: sums_and_products. - @return: pipeline1 - """ - fname = os.path.join("..", "samplecode", "script_1_sum_and_products.py") - with open(fname, "r") as f: - sumsprodsrc = f.read() - - sums_and_products = self.create_method_from_string( - 'sums_and_products', - sumsprodsrc, - user, - ['pairs'], - ['sums_and_products']) - sums_and_products.reusable = Method.REUSABLE - sums_and_products.save() - sums_and_products.clean() - with transaction.atomic(): - family = PipelineFamily(name='sums and products', user=user) - family.save() - family.clean() - family.grant_everyone_access() - - pipeline1 = family.members.create(revision_name='sums and products', - user=user) - pipeline1.clean() - pipeline1.grant_everyone_access() - - self.next_step_num = 1 - self.next_output_num = 1 - input1 = pipeline1.inputs.create(dataset_name='pairs', - dataset_idx=1) - step1_1 = self.create_step(pipeline1, sums_and_products, input1) - self.create_cable(step1_1, pipeline1) - pipeline1.create_outputs() - self.set_position([input1, - step1_1, - pipeline1.outputs.first()]) - pipeline1.complete_clean() - return pipeline1 - - def build_MC(self): - """Create a MiCall pipeline - Generate two data sets as input to the pipeline. - Execute the pipeline. - """ - user = User.objects.first() - pipeline1 = self.create_MC_pipelines(user) - - # First, read the two data sets - tklst = ["1234A-V3LOOP_S1_L001_R1_001.fastq", - "1234A-V3LOOP_S1_L001_R2_001.fastq"] - datalst = self._rdfilelst([os.path.join(self.src_dir, tk) for tk in tklst]) - datasetlst = [] - for dataname, datacontent in zip(tklst, datalst): - ds_file = ContentFile(datacontent) - dset = Dataset(user=user, - name=dataname, - MD5_checksum=compute_md5(ds_file)) - dset.dataset_file.save(name=dataname, content=ds_file) - dset.save() - dset.clean() - dset.grant_everyone_access() - datasetlst.append(dset) - - # now execute the pipeline - run = Manager.execute_pipeline(user=user, - pipeline=pipeline1, - inputs=datasetlst).get_last_run() - run.collect_garbage() # Delete sandbox directories - - def create_MC_pipelines(self, user): - """ Create a MiCall Demo pipeline. - - @return: (pipeline1) - """ - # various package dependencies - # NOTE: The SINGLE __init__.py file is handled as a special case in the dependencies. - init_file_name = "micall.__init__.py" - - # files that are required for individual driver modules - loggingdeps = ["micall.core.miseq_logging.py"] - configdeps = ["micall.core.project_config.py", - "micall.core.project_scoring.json", "micall.core.projects.json"] - # dependencies for the micall.utils.externals module - externaldeps = ["micall.utils.externals.py"] - # dependencies for the micall.utils.translation module - translationdeps = ["micall.utils.translation.py"] - - sam2alndeps = ["micall.core.sam2aln.py"] - aln2countsdeps = loggingdeps + configdeps + translationdeps + ["micall.core.aln2counts.py"] - - g2pdeps = ["micall.g2p.pssm_lib.py", "micall.g2p.g2p_fpr.txt", "micall.g2p.g2p.matrix"] - # now the pipeline step dependencies - prelim_deplst = loggingdeps + configdeps + externaldeps - remap__deplst = (loggingdeps + configdeps + externaldeps + - translationdeps + sam2alndeps + ["micall.core.prelim_map.py"]) - sam2al_deplst = [] - aln2co_deplst = loggingdeps + configdeps + translationdeps - - sam_g2p_deplst = translationdeps + sam2alndeps + g2pdeps - - coverage_deplst = configdeps + aln2countsdeps - # NOTE: The order of this list is significant: the order determines - # the step number in the pipeline. A step number i must read all of its inputs - # from a step number < i or a pipeline input. - # (stepname, codefile, inputlst, outputlst, deplist) - methlst = [("prelim", "micall.core.prelim_map.py", - ["forward", "reverse"], ["prelim_out"], - frozenset(prelim_deplst)), - ("remap", "micall.core.remap.py", - ["forward", "reverse", "prelim_out"], - ["remap_csv", "remap_counts", "remap_conseq", - "unmappedfor", "unmappedrev"], - frozenset(remap__deplst)), - ("sam2aln", "micall.core.sam2aln.py", - ["remap_csv"], - ["aligned", "conseq_insertions", "failed_read"], - frozenset(sam2al_deplst)), - ("aln2counts", "micall.core.aln2counts.py", - ["aligned"], - ["nuc", "amino", "coord_ins", "conseq", "failed_align", - "nuc_variants"], - frozenset(aln2co_deplst)), - ("sam_g2p", "micall.g2p.sam_g2p.py", - ["remap_csv", "nuc"], - ["g2p", "g2p_summary"], - frozenset(sam_g2p_deplst)), - ("coverage_plots", "micall.core.coverage_plots.py", - ["amino"], - ["coverage_scores_csv", "coverage_maps_tar"], - frozenset(coverage_deplst))] - # need the union of all code resources - allcodeset = set([init_file_name] + [k[1] for k in methlst]).union(*[k[4] for k in methlst]) - codedct = dict([(dp, self.read_dep_resource(user, self.src_dir, dp)) for dp in allcodeset]) - - init_code_resource = codedct[init_file_name][0] - # create all of the methods and keep them in a dictionary - methdct = {} - # also create output name and input name lookup dicts - outpnamedct = {} - inpnamedct = {} - for methname, methfile, inputs, outputs, depset in methlst: - newmeth = methdct[methname] = self.create_method_from_resource(methname, - codedct[methfile][0], - user, - inputs, outputs, - [codedct[dpname] for dpname in depset], - init_code_resource) - newmeth.reusable = Method.REUSABLE - newmeth.save() - newmeth.clean() - # - for n, opname in enumerate(outputs, 1): - if opname in outpnamedct: - raise RuntimeError("Output names are not unique!") - outpnamedct[opname] = (methname, n) - - # NOTE: input names are not unique globally, only at the method level: - # therefore have a separate dict for each method - inpnamedct[methname] = dict(enumerate(outputs, 1)) - - # we now have our methods and have made sure that each one can fulfil - # its inputs from a pipeline input or a previous step. - # now build the pipeline: - # a) each method is turned into a pipeline step - # b) the cables are connected between steps - # c) connect output cables to all step outputs not accounted for (i.e. 'dangling') - with transaction.atomic(): - family = PipelineFamily(name='MiCallDemo', user=user) - family.save() - family.clean() - family.grant_everyone_access() - - pipeline1 = family.members.create(revision_name='MiCall Demo 2016-09', - user=user) - pipeline1.clean() - pipeline1.grant_everyone_access() - - self.next_output_num = 1 - # this pipeline has two inputs - input1 = pipeline1.inputs.create(dataset_name='forward', dataset_idx=1) - input2 = pipeline1.inputs.create(dataset_name='reverse', dataset_idx=2) - - stepdct = {} - for stepnum, methname in enumerate([k[0] for k in methlst], 1): - mymeth = methdct[methname] - newstep = stepdct[methname] = pipeline1.steps.create(transformation=mymeth, - name=mymeth.family.name, - step_num=stepnum) - newstep.clean() - - # go through the steps in order, wiring up all inputs - # keep track of all unresolved inputs - missing_inp_set = set() - # Also keep track of outputs that are not connected to other steps as inputs. - # Create set of all output names, then strike them off as we connect outputs up. - # If we have any of these left over once we have connected all steps, we assume - # that the remaining 'dangling' outputs are pipeline outputs. - dangling_outp_set = set(["forward", "reverse"]).union(*[set(k[3]) for k in methlst]) - curinpdct = {"forward": (input1, 0), "reverse": (input2, 0)} - for methname, inplst, outplst in [(k[0], k[2], k[3]) for k in methlst]: - # satisfy all input requirements of this step by name matching - curstep = stepdct[methname] - for curinpnum, curinpname in enumerate(inplst, 1): - if curinpname in curinpdct: - src_obj, src_outpnum = curinpdct[curinpname] - self.create_internal_cable(src_obj, src_outpnum, curstep, curinpnum) - if curinpname in dangling_outp_set: - dangling_outp_set.remove(curinpname) - else: - missing_inp_set.add(curinpname) - # --now add this step's outputs to the list of possible inputs for the next step - for outpnum, outpname in enumerate(outplst, 1): - if outpname in curinpdct: - raise RuntimeError("Data source names are not unique!") - curinpdct[outpname] = (curstep, outpnum) - - # --missing_inp_set must be empty, or we have failed to account for all inputs - if missing_inp_set: - raise RuntimeError("MiCallDemo: failed to create pipeline. Missing inputs:" + - ", ".join(missing_inp_set)) - - method_names = map(itemgetter(0), methlst) - dangling_outputs = [] - for output_name in dangling_outp_set: - method_name, step_output_index = outpnamedct[output_name] - method_num = method_names.index(method_name) - dangling_outputs.append((method_num, step_output_index, output_name)) - dangling_outputs.sort() - # all dangling outputs are assumed to be pipeline outputs - for opnum, dangling_output in enumerate(dangling_outputs, 1): - method_num, step_output_index, output_name = dangling_output - method_name, _ = outpnamedct[output_name] - self.create_PL_output(pipeline1, - stepdct[method_name], - step_output_index, - output_name, - opnum) - - pipeline1.create_outputs() - - outputs = list(pipeline1.outputs.order_by('dataset_idx')) - self.set_position([input1, input2] + - [stepdct[k[0]] for k in methlst] + - outputs[:1]) - n = len(outputs) - y = outputs[0].y - for i, output in enumerate(outputs, 1): - output.x = 0.25 + 0.75*float(i)/(n+1) - output.y = y - output.save() - - pipeline1.complete_clean() - return pipeline1 - - -class RemovalTestEnvironmentBuilder(FixtureBuilder): - def get_name(self): - return 'removal.json' - - def build(self): - tools.create_removal_test_environment() - - -class ExecuteTestsBuilder(FixtureBuilder): - def get_name(self): - return "execute_tests.json" - - def build(self): - sandbox.tests.execute_tests_environment_setup(self) - - -class RunApiTestsEnvironmentBuilder(FixtureBuilder): - def get_name(self): - return "run_api_tests.json" - - def build(self): - sandbox.tests.execute_tests_environment_setup(self) - - # Define pipeline containing two steps with the same method + pipeline input - self.pX_2 = pipeline.models.Pipeline(family=self.pf, revision_name="pX_revision_2", - revision_desc="X2", user=self.myUser) - self.pX_2.save() - self.X1_in = self.pX_2.create_input(compounddatatype=self.pX_in_cdt, dataset_name="pX_in", dataset_idx=1) - self.step_X1 = self.pX_2.steps.create(transformation=self.mA, step_num=1) - self.step_X2 = self.pX_2.steps.create(transformation=self.mA, step_num=2) - - # Use the SAME custom cable from pipeline input to steps 1 and 2 - self.cable_X1_A1 = self.step_X1.cables_in.create(dest=self.mA_in, source_step=0, source=self.X1_in) - self.wire1 = self.cable_X1_A1.custom_wires.create(source_pin=self.pX_in_cdtm_2, dest_pin=self.mA_in_cdtm_2) - self.wire2 = self.cable_X1_A1.custom_wires.create(source_pin=self.pX_in_cdtm_3, dest_pin=self.mA_in_cdtm_1) - self.cable_X1_A2 = self.step_X2.cables_in.create(dest=self.mA_in, source_step=0, source=self.X1_in) - self.wire3 = self.cable_X1_A2.custom_wires.create(source_pin=self.pX_in_cdtm_2, dest_pin=self.mA_in_cdtm_2) - self.wire4 = self.cable_X1_A2.custom_wires.create(source_pin=self.pX_in_cdtm_3, dest_pin=self.mA_in_cdtm_1) - - # POCs: one is trivial, the second uses custom outwires - # Note: by default, create_outcables assumes the POC has the CDT of the source (IE, this is a TRIVIAL cable) - self.outcable_1 = self.pX_2.create_outcable(output_name="pX_out_1", - output_idx=1, - source_step=1, - source=self.mA_out) - self.outcable_2 = self.pX_2.create_outcable(output_name="pX_out_2", - output_idx=2, - source_step=2, - source=self.mA_out) - - # Define CDT for the second output (first output is defined by a trivial cable) - self.pipeline_out2_cdt = CompoundDatatype(user=self.myUser) - self.pipeline_out2_cdt.save() - self.out2_cdtm_1 = self.pipeline_out2_cdt.members.create( - column_name="c", - column_idx=1, - datatype=self.int_dt) - self.out2_cdtm_2 = self.pipeline_out2_cdt.members.create( - column_name="d", - column_idx=2, - datatype=self.string_dt) - self.out2_cdtm_3 = self.pipeline_out2_cdt.members.create( - column_name="e", - column_idx=3, - datatype=self.string_dt) - - # Second cable is not trivial - we assign the new CDT to it - self.outcable_2.output_cdt = self.pipeline_out2_cdt - self.outcable_2.save() - - # Define custom outwires to the second output (Wire twice from cdtm 2) - self.outwire1 = self.outcable_2.custom_wires.create(source_pin=self.mA_out_cdtm_1, dest_pin=self.out2_cdtm_1) - self.outwire2 = self.outcable_2.custom_wires.create(source_pin=self.mA_out_cdtm_2, dest_pin=self.out2_cdtm_2) - self.outwire3 = self.outcable_2.custom_wires.create(source_pin=self.mA_out_cdtm_2, dest_pin=self.out2_cdtm_3) - - # Have the cables define the TOs of the pipeline - self.pX_2.create_outputs() - - # Run this pipeline. - Manager.execute_pipeline(self.myUser, self.pX_2, [self.dataset]) - - -class RunComponentTooManyChecksEnvironmentBuilder(FixtureBuilder): - def get_name(self): - return "run_component_too_many_checks.json" - - def build(self): - tools.create_word_reversal_environment(self) - - # Set up and run a Pipeline that throws away its intermediate data. - self.two_step_pl = tools.make_first_pipeline("Two-step pipeline", - "Toy pipeline for testing data check cleaning of RunSteps.", - self.user_bob) - tools.create_linear_pipeline(self.two_step_pl, [self.method_noop_wordbacks, self.method_noop_wordbacks], - "data", "samedata") - first_step = self.two_step_pl.steps.get(step_num=1) - first_step.add_deletion(self.method_noop_wordbacks.outputs.first()) - first_step.save() - - Manager.execute_pipeline(self.user_bob, self.two_step_pl, [self.dataset_wordbacks], - groups_allowed=[everyone_group()]) - - # The second one's second step will have to recover its first step. (Its input cable is trivial - # and is able to reuse the input cable from the first Pipeline's second step.) - self.following_pl = tools.make_first_pipeline( - "Pipeline that will follow the first", - "Toy pipeline that will need to recover its first step when following the above.", - self.user_bob - ) - tools.create_linear_pipeline(self.following_pl, [self.method_noop_wordbacks, self.method_reverse], - "data", "reversed_data") - first_step = self.following_pl.steps.get(step_num=1) - first_step.add_deletion(self.method_noop_wordbacks.outputs.first()) - first_step.save() - - following_run = Manager.execute_pipeline(self.user_bob, self.following_pl, [self.dataset_wordbacks], - groups_allowed=[everyone_group()]).get_last_run() - second_step = following_run.runsteps.get(pipelinestep__step_num=2) - assert(second_step.invoked_logs.count() == 3) - - # FIXME are there other files that aren't properly being removed? - if hasattr(self, "words_datafile"): - os.remove(self.words_datafile.name) - - -class RunPipelinesRecoveringReusedStepEnvironmentBuilder(FixtureBuilder): - """ - Setting up and running two pipelines, where the second one reuses and - then recovers a step from the first. - """ - - def get_name(self): - return "run_pipelines_recovering_reused_step.json" - - def build(self): - tools.create_eric_martin_test_environment(self) - tools.create_sandbox_testing_tools_environment(self) - - p_one = tools.make_first_pipeline("p_one", "two no-ops", self.myUser) - p_one.family.grant_everyone_access() - p_one.grant_everyone_access() - tools.create_linear_pipeline(p_one, [self.method_noop, self.method_noop], "p_one_in", "p_one_out") - p_one.create_outputs() - p_one.save() - # Mark the output of step 1 as not retained. - p_one.steps.get(step_num=1).add_deletion(self.method_noop.outputs.first()) - - p_two = tools.make_first_pipeline("p_two", "one no-op then one trivial", self.myUser) - p_two.family.grant_everyone_access() - p_two.grant_everyone_access() - tools.create_linear_pipeline(p_two, [self.method_noop, self.method_trivial], "p_two_in", "p_two_out") - p_two.create_outputs() - p_two.save() - # We also delete the output of step 1 so that it reuses the existing ER we'll have - # create for p_one. - p_two.steps.get(step_num=1).add_deletion(self.method_noop.outputs.first()) - - # Set up a words dataset. - tools.make_words_dataset(self) - - Manager.execute_pipeline(user=self.user_bob, pipeline=p_one, inputs=[self.dataset_words], - groups_allowed=[everyone_group()], - name="RunOne", description="Bob runs p_one") - - Manager.execute_pipeline( - user=self.user_bob, - pipeline=p_two, - inputs=[self.dataset_words], - groups_allowed=[everyone_group()], - name="RunOne", - description="Bob runs p_two, which tries to reuse part of run_one but ultimately needs to recover" - ) - - -class ExecuteResultTestsRMEnvironmentBuilder(FixtureBuilder): - """ - Execution tests using tools.create_sequence_manipulation_environment. - """ - - def get_name(self): - return "execute_result_tests_rm.json" - - def build(self): - tools.create_sequence_manipulation_environment(self) - - # Many tests use this run. - Manager.execute_pipeline(self.user_alice, self.pipeline_complement, [self.dataset_labdata]) - - # An identically-specified second run that reuses the stuff from the first. - Manager.execute_pipeline(self.user_alice, self.pipeline_complement, [self.dataset_labdata]) - - # A couple more runs, used by another test. - Manager.execute_pipeline(self.user_alice, self.pipeline_reverse, [self.dataset_labdata]) - - Manager.execute_pipeline(self.user_alice, self.pipeline_revcomp, [self.dataset_labdata]) - - # This is usually done in tools.destroy_sequence_manipulation_environment. - if os.path.exists(self.datafile.name): - os.remove(self.datafile.name) - - -class ExecuteDiscardedIntermediateTestsRMEnvironmentBuilder(FixtureBuilder): - """ - Creates a fixture used by execution tests involving discarded intermediate data. - """ - - def get_name(self): - return "execute_discarded_intermediate_tests_rm.json" - - def build(self): - tools.create_sequence_manipulation_environment(self) - - # This run will discard intermediate data. - Manager.execute_pipeline(self.user_alice, self.pipeline_revcomp_v2, [self.dataset_labdata]) - - # This is usually done in tools.destroy_sequence_manipulation_environment. - if os.path.exists(self.datafile.name): - os.remove(self.datafile.name) - - -class FindDatasetsBuilder(FixtureBuilder): - """For testing the tools that find datasets in a sandbox.""" - - def get_name(self): - return "find_datasets.json" - - def build(self): - tools.create_word_reversal_environment(self) - - self.setup_simple_pipeline() - self.setup_twostep_pipeline() - self.setup_nested_pipeline() - - def setup_nested_pipeline(self): - # A two-step pipeline with custom cable wires at each step. - self.pipeline_nested = tools.make_first_pipeline( - "nested pipeline", - "a pipeline with a sub-pipeline", - self.user_bob) - - transforms = [self.method_noop_backwords, self.pipeline_twostep, self.method_noop_backwords] - tools.create_linear_pipeline(self.pipeline_nested, - transforms, - "data", - "unchanged_data") - cable = self.pipeline_nested.steps.get(step_num=3).cables_in.first() - tools.make_crisscross_cable(cable) - self.pipeline_nested.create_outputs() - self.pipeline_nested.complete_clean() - - def setup_twostep_pipeline(self): - """ - (drow,word) (word,drow) (word,drow) (drow,word) (drow,word) (drow,word) - _____________ ______________ - [o]====<>====|o o|=====<>=====|o o|============[o] - | reverse | | noop | - |_____________| |______________| - """ - # A two-step pipeline with custom cable wires at each step. - self.pipeline_twostep = tools.make_first_pipeline( - "two-step pipeline", - "a two-step pipeline with custom cable wires at each step", - self.user_bob) - self.pipeline_twostep.create_input(compounddatatype=self.cdt_backwords, dataset_name="words_to_reverse", - dataset_idx=1) - - methods = [self.method_reverse, self.method_noop_backwords] - for i, _method in enumerate(methods): - step = self.pipeline_twostep.steps.create(transformation=methods[i], step_num=i+1) - if i == 0: - source = self.pipeline_twostep.inputs.first() - else: - source = methods[i-1].outputs.first() - cable = step.cables_in.create(source_step=i, - source=source, - dest=methods[i].inputs.first()) - tools.make_crisscross_cable(cable) - - cable = self.pipeline_twostep.create_outcable( - output_name="reversed_words", - output_idx=1, - source_step=2, - source=methods[-1].outputs.first()) - - self.pipeline_twostep.create_outputs() - self.pipeline_twostep.complete_clean() - - def setup_simple_pipeline(self): - # A simple, one-step pipeline, which does nothing. - self.pipeline_noop = tools.make_first_pipeline("simple pipeline", "a simple, one-step pipeline", - self.user_bob) - tools.create_linear_pipeline( - self.pipeline_noop, - [self.method_noop], - "lab_data", "complemented_lab_data") - self.pipeline_noop.create_outputs() - - class ContainerRunBuilder(FixtureBuilder): """For testing the tools that find datasets in a sandbox.""" @@ -1294,22 +282,6 @@ class Command(BaseCommand): help = "Update test fixtures by running scripts and dumping test data." def handle(self, *args, **options): - EMSandboxTestEnvironmentBuilder().run() - ArchiveTestEnvironmentBuilder().run() - ArchiveNoRunsTestEnvironmentBuilder().run() - DeepNestedRunBuilder().run() - SimpleRunBuilder().run() - RemovalTestEnvironmentBuilder().run() - RunApiTestsEnvironmentBuilder().run() - RunComponentTooManyChecksEnvironmentBuilder().run() - RunPipelinesRecoveringReusedStepEnvironmentBuilder().run() - ExecuteResultTestsRMEnvironmentBuilder().run() - ExecuteDiscardedIntermediateTestsRMEnvironmentBuilder().run() - RestoreReusableDatasetBuilder().run() - ExecuteTestsBuilder().run() - FindDatasetsBuilder().run() - RestoreReusableDatasetBuilder().run() - DemoBuilder().run() ContainerRunBuilder().run() self.stdout.write('Done.') diff --git a/kive/sandbox/execute.py b/kive/sandbox/execute.py deleted file mode 100644 index 37caec771..000000000 --- a/kive/sandbox/execute.py +++ /dev/null @@ -1,2631 +0,0 @@ -"""Code that is responsible for the execution of Pipelines.""" - -from collections import defaultdict -import logging -import os -import random -import tempfile -import time -import itertools -# import pwd - -from django.utils import timezone -from django.db import transaction, OperationalError, InternalError -from django.contrib.auth.models import User -from django.conf import settings - -from archive.models import RunStep, Run, ExecLog, RunSIC, RunCable, RunComponent, RunOutputCable -from constants import dirnames, extensions, runcomponentstates -import file_access_utils -from container.models import Container -from librarian.models import Dataset, ExecRecord -import pipeline.models -from method.models import Method, DockerImage -from datachecking.models import IntegrityCheckLog -from fleet.exceptions import StopExecution -from file_access_utils import copy_and_confirm, FileCreationError - -logger = logging.getLogger("Sandbox") - -sandbox_prefix = "user{}_run{}_" -# This is used by the fleet Manager when cleaning up. -sandbox_glob = sandbox_prefix.format("*", "*") + "*" - - -class Sandbox: - """ - A Sandbox is the environment in which a Pipeline is run. It contains - all the information necessary to run the Pipeline, including the code - for all steps of the Pipeline, and the data to feed in. The Sandbox keeps - track of a single Run of the Pipeline it was created with. - - Note that Sandboxes are single-use; that is, a Pipeline may only be run - once in a Sandbox. To run the same Pipeline again, you must create a new - Sandbox. - """ - # dataset_fs_map: maps Dataset to a FS path: the path where a data - # file would be if it were created (Whether or not it is there) - # If the path is None, the Dataset is on the DB. - - # socket_map: maps (run, generator, socket) to Datasets. - # A generator is a cable, or a pipeline step. A socket is a TI/TO. - # If the generator is none, the socket is a pipeline input. - # This will be used to look up inputs when running a pipeline. - - # ps_map: maps PS to (path, RunStep of PS): the path tells you - # the directory that the PS would have been run in - # (whether or not it was): the RunStep tells you what inputs are - # needed (Which in turn will lead back to an dataset_fs_map lookup), - # and allows you to fill it in on recovery. - - # queue_for_processing: list of tasks that are ready to be processed, i.e. - # all of the required inputs are available and ready to go. - - # step_execute_info: table of RunStep "bundles" giving all the information - # necessary to process a RunStep. - - # cable_map maps cables to ROC/RSIC. - - def __init__(self, run): - """ - Sets up a sandbox environment to run a Pipeline: space on - the file system, along with dataset_fs_map/socket_map/etc. - - INPUTS - run A Run object to fill in (e.g. if we're starting this using the fleet); - - PRECONDITIONS - run.inputs must have real data - """ - self.run = run - user = run.user - my_pipeline = run.pipeline - inputs = [x.dataset for x in run.inputs.order_by("index")] - sandbox_path = run.sandbox_path - - self.logger = logger - # logging.getLogger(self.__class__.__name__) - self.user = user - self.pipeline = my_pipeline - self.inputs = inputs - self.dataset_fs_map = {} - self.socket_map = {} - self.cable_map = {} - self.ps_map = {} - self.pipeline.check_inputs(self.inputs) - - # Determine a sandbox path, and input/output directories for - # top-level Pipeline. - self.sandbox_path = sandbox_path or tempfile.mkdtemp( - prefix=sandbox_prefix.format(self.user, self.run.pk), - dir=file_access_utils.create_sandbox_base_path()) - - self.run.sandbox_path = self.sandbox_path - self.run.save() - - in_dir = os.path.join(self.sandbox_path, dirnames.IN_DIR) - self.out_dir = os.path.join(self.sandbox_path, dirnames.OUT_DIR) - - self.logger.debug("initializing maps") - for i, pipeline_input in enumerate(inputs, start=1): - corresp_pipeline_input = self.pipeline.inputs.get(dataset_idx=i) - self.socket_map[(self.run, None, corresp_pipeline_input)] = pipeline_input - self.dataset_fs_map[pipeline_input] = os.path.join( - in_dir, - "run{}_{}".format(self.run.pk, corresp_pipeline_input.pk)) - - # Make the sandbox directory. - self.logger.debug("file_access_utils.set_up_directory({})".format(self.sandbox_path)) - file_access_utils.configure_sandbox_permissions(self.sandbox_path) - file_access_utils.set_up_directory(self.sandbox_path) - file_access_utils.set_up_directory(in_dir) - file_access_utils.set_up_directory(self.out_dir) - file_access_utils.configure_sandbox_permissions(self.out_dir) - - # Queue of RunSteps/RunCables to process. - self.queue_for_processing = [] - - # PipelineSteps and PipelineCables "bundled" with necessary information for running them. - # This will be used when it comes time to finish these cables, either as a first execution or as a recovery. - # Keys are (run, generator) pairs. - self.step_execute_info = {} - self.cable_execute_info = {} - - # A table of RunSteps/RunCables completed. - # self.tasks_completed = {} - - # A table keyed by Datasets, whose values are lists of the RunSteps/RunCables waiting on them. - self.tasks_waiting = defaultdict(list) - - # The inverse table to the above: the keys are RunSteps/RunCables waiting on recovering Datasets, - # and the values are all of the Datasets they're waiting for. - self.waiting_for = {} - - # For each sub-pipeline, we track how many of their input cables have completed. - self.sub_pipeline_cable_tracker = {} - - class RunInputEmptyException(Exception): - """ - An exception raised when a cable encounters a Run input that has no data. - """ - pass - - def step_xput_path(self, runstep, transformationxput, step_run_dir): - """Path in Sandbox for PipelineStep TransformationXput.""" - file_suffix = extensions.RAW if transformationxput.is_raw() else extensions.CSV - file_name = "step{}_{}.{}".format(runstep.step_num, transformationxput.dataset_name, file_suffix) - - if transformationxput.is_input: - xput_dir = dirnames.IN_DIR - else: - xput_dir = dirnames.OUT_DIR - return os.path.join(step_run_dir, xput_dir, file_name) - - def register_dataset(self, dataset, location): - """Set the location of a Dataset on the file system. - - If the Dataset is already in the Sandbox (ie. it is in - dataset_fs_map), do not update the existing location. - - INPUTS - dataset Dataset to register - location file path of dataset in the Sandbox - """ - try: - self.dataset_fs_map[dataset] = self.dataset_fs_map[dataset] or location - except KeyError: - self.dataset_fs_map[dataset] = location - - def find_dataset(self, dataset): - """Find the location of a Dataset on the file system. - - INPUTS - dataset Dataset to locate - - OUTPUTS - location the path of dataset in the Sandbox, - or None if it's not there - """ - try: - location = self.dataset_fs_map[dataset] - except KeyError: - self.logger.debug("Dataset {} is not in the Sandbox".format(dataset)) - return None - - return (location - if location and file_access_utils.file_exists(location) - else None) - - def update_cable_maps(self, runcable, output_dataset, output_path): - """Update maps after cable execution. - - INPUTS - runcable RunCable created for cable execution - output_dataset Dataset output by cable - output_path where the cable wrote its output - """ - self.register_dataset(output_dataset, output_path) - cable = runcable.component - self.socket_map[(runcable.parent_run, cable, cable.dest)] = output_dataset - self.cable_map[(runcable.parent, cable)] = runcable - - def update_step_maps(self, runstep, step_run_dir, output_paths): - """Update maps after pipeline step execution. - - INPUTS - runstep RunStep responsible for execution - step_run_dir directory where execution was done - output_paths paths where RunStep outputs were written, - ordered by index - """ - pipelinestep = runstep.component - my_run = runstep.run - self.ps_map[(my_run, pipelinestep)] = (step_run_dir, runstep) - execrecordouts = runstep.execrecord.execrecordouts - - for i, step_output in enumerate(pipelinestep.transformation.outputs.order_by("dataset_idx")): - corresp_ero = execrecordouts.get(generic_output=step_output) - corresp_dataset = corresp_ero.dataset - self.register_dataset(corresp_dataset, output_paths[i]) - - # This pipeline step, with the downstream TI, maps to corresp_dataset - self.socket_map[(runstep.parent_run, pipelinestep, step_output)] = corresp_dataset - - def _register_missing_output(self, output_dataset, execlog, start_time, end_time): - """Create a failed ContentCheckLog for missing cable output - - INPUTS - output_dataset Dataset cable was supposed to output - execlog ExecLog for cable execution which didn't produce - output - start_time time when we started checking for missing output - end_time time when we finished checking for missing output - """ - self.logger.error("File doesn't exist - creating CCL with BadData") - ccl = output_dataset.content_checks.create(start_time=start_time, end_time=end_time, execlog=execlog, - user=self.user) - ccl.add_missing_output() - - def first_generator_of_dataset(self, dataset_to_find, curr_run=None): - """ - Find the (run, generator) pair which first produced a Dataset. - If generator is None, it indicates the socket is a Pipeline input. If - both generator and run are None, it means the dataset wasn't found in the - Pipeline. - """ - # Is this a sub-run or a top-level run? - if curr_run is None: - # This is a top-level run. Set curr_run accordingly. - curr_run = self.run - pipeline = curr_run.pipeline - - # First check if the dataset we're looking for is a Pipeline input. - if curr_run == self.run: - for socket in pipeline.inputs.order_by("dataset_idx"): - key = (self.run, None, socket) - if key in self.socket_map and self.socket_map[key] == dataset_to_find: - return (self.run, None) - - # If it's not a pipeline input, check all the steps. - steps = curr_run.runsteps.all() - steps = sorted(steps, key=lambda step: step.pipelinestep.step_num) - - for step in steps: - # First check if the dataset is an input to this step. In that case, it - # had to come in from a nontrivial cable (since we're checking the - # steps in order, and we already checked everything produced prior to - # this step). - pipelinestep = step.pipelinestep - for socket in pipelinestep.transformation.inputs.order_by("dataset_idx"): - generator = pipelinestep.cables_in.get(dest=socket) - key = (curr_run, generator, socket) - if key in self.socket_map and self.socket_map[key] == dataset_to_find: - return (curr_run, generator) - - # If it wasn't an input to this step, but this step is a sub-pipeline, - # it might be somewhere within the sub-pipeline. Search recursively. - if hasattr(step, "child_run") and step.child_run is not None: - run, generator = self.first_generator_of_dataset(dataset_to_find, step.child_run) - if run is not None: - - # Did we find it somewhere inside the sub-Pipeline? - if generator is not None: - return (run, generator) - - # If it was an input to the sub-Pipeline, we need the cable leading in. - # May 23, 2014: I think this is redundant as this will get caught when - # looking at the cables feeding into this step. - # else: - # generator = pipelinestep.cables_in.get(dest=socket) - # return (curr_run, generator) - - # Now check if it's an output from this step. - generator = pipelinestep - for socket in pipelinestep.transformation.outputs.order_by("dataset_idx"): - key = (curr_run, generator, socket) - if key in self.socket_map and self.socket_map[key] == dataset_to_find: - return (curr_run, generator) - - # Finally, check if it's at the end of a nontrivial Pipeline output cable. - for outcable in pipeline.outcables.order_by("output_idx"): - socket = outcable.dest - key = (curr_run, outcable, socket) - if key in self.socket_map and self.socket_map[key] == dataset_to_find: - return (curr_run, outcable) - - # If we're here, we didn't find it. - return (None, None) - - #### - # Code to execute code in an MPI environment. - - def enqueue_runnable_tasks(self, data_newly_available): - """ - Function that queues steps/outcables that are ready to run now that new data is available. - """ - for dataset in data_newly_available: - assert dataset.has_data() or self.find_dataset(dataset) is not None - - # First, get anything that was waiting on this data to proceed. - taxiing_for_takeoff = [] - for dataset in data_newly_available: - if dataset in self.tasks_waiting: - # Notify all tasks waiting on this dataset that it's now available. - # Trigger any tasks for which that was the last piece it was waiting on. - for task in self.tasks_waiting[dataset]: - self.waiting_for[task].remove(dataset) - if len(self.waiting_for[task]) == 0: - # Add this to the list of things that are ready to go. - taxiing_for_takeoff.append(task) - - # Remove this entry from self.tasks_waiting. - self.tasks_waiting.pop(dataset) - - self.queue_for_processing = self.queue_for_processing + taxiing_for_takeoff - - def advance_pipeline(self, task_completed=None, run_to_advance=None, incables_completed=None, - steps_completed=None, outcables_completed=None): - """ - Proceed through a pipeline, seeing what can run now that a step or cable has just completed. - - Note that if a sub-pipeline of the pipeline finishes, we report that the parent runstep - has finished, not the cables. - - If task_completed is specified, that indicates that a new RunComponent has just finished - (i.e. by the fleet), so we attempt to advance the Pipeline. - - If run_to_advance is specified, it means this is a recursive call, attempting to advance - a sub-Pipeline given the new stuff that has been finished so far (which is passed on - through the parameters incables_completed, steps_completed, and outcables_completed). - - PRE: - at most one of run_to_advance and task_completed may not be None. - if task_completed is not None, it is finished and successful. - """ - assert (type(task_completed) in (RunStep, - RunSIC, - RunOutputCable) or - task_completed is None) - assert not (run_to_advance is not None and task_completed is not None) - - incables_completed = incables_completed or [] - steps_completed = steps_completed or [] - outcables_completed = outcables_completed or [] - - run_to_resume = self.run - if run_to_advance: - assert run_to_advance.top_level_run == self.run - run_to_resume = run_to_advance - - if task_completed is None and run_to_advance is None: - self.logger.debug('Starting run "%s"', self.run) - elif task_completed is not None: - self.logger.debug('Advancing run "%s" after completion of task %s (coordinates: %s)', - self.run, - task_completed, - task_completed.get_coordinates()) - else: # run_to_advance is not None - self.logger.debug('Advancing sub-run "%s" (coordinates %s) of pipeline %s', - run_to_resume, run_to_resume.get_coordinates(), self.run) - - if task_completed is None and not run_to_resume.has_started(): - run_to_resume.start(save=True) - - # Refresh the run plan, unless this is a recursive call that starts a new sub-Pipeline. - if not run_to_advance: - self.run_plan = RunPlan() - self.run_plan.load(self.run, self.inputs) - self.run_plan.find_consistent_execution() - - pipeline_to_resume = run_to_resume.pipeline - - if run_to_resume != self.run: - assert run_to_resume.top_level_run == self.run - - sandbox_path = self.sandbox_path - if run_to_resume != self.run: - sandbox_path = (self.step_execute_info[(run_to_resume.parent_runstep.run, - run_to_resume.parent_runstep.pipelinestep)] - .step_run_dir) - - # Update our lists of components completed. - step_nums_completed = [] - if type(task_completed) == RunSIC: - assert task_completed.dest_runstep.pipelinestep.is_subpipeline() - incables_completed.append(task_completed) - elif type(task_completed) == RunStep: - steps_completed.append(task_completed) - elif type(task_completed) == RunOutputCable: - outcables_completed.append(task_completed) - elif task_completed is None and run_to_advance is None: - # This indicates that the only things accessible are the inputs. - step_nums_completed.append(0) - - step_nums_completed += [x.step_num for x in steps_completed if x.run == run_to_resume] - - # A tracker for whether everything is complete or not. - all_complete = True - - # Go through steps in order, looking for input cables pointing at the task(s) that have completed. - # If task_completed is None, then we are starting the pipeline and we look at the pipeline inputs. - for step in pipeline_to_resume.steps.order_by("step_num"): - curr_RS = run_to_resume.runsteps.filter(pipelinestep=step).first() - assert curr_RS is not None - - # If this is already running, we skip it, unless it's a sub-Pipeline. - if curr_RS.is_running(): - if not step.is_subpipeline(): - # This is a non-sub-run already in progress, so we leave it. - all_complete = False - continue - - # At this point, we know this is a sub-Pipeline, and is possibly waiting - # for one of its input cables to finish. - if type(task_completed) == RunSIC: - feeder_RSICs = curr_RS.RSICs.filter(pk__in=[x.pk for x in incables_completed]) - if not feeder_RSICs.exists(): - # This isn't one of the RunSICs for this sub-Run. - all_complete = False - continue - else: - self.sub_pipeline_cable_tracker[curr_RS].difference_update(set(feeder_RSICs)) - if len(self.sub_pipeline_cable_tracker[curr_RS]) != 0: - # Not all of the cables are done yet. - all_complete = False - continue - - else: - # Look in the lists of tasks completed. Do any of them belong to this sub-run? - complete_subtask_exists = False - for task in itertools.chain(incables_completed, steps_completed, outcables_completed): - task_coords = task.get_coordinates() - curr_step_coords = curr_RS.get_coordinates() - if task_coords[0:len(curr_step_coords)] == curr_step_coords: - complete_subtask_exists = True - break - - if not complete_subtask_exists: - continue - - # Having reached here, we know that task_completed was either: - # - the last RunSIC the sub-Pipeline was waiting on, or - # - a task belonging to the sub-Run, - # so we can advance the sub-Run and update the lists of components - # completed. - incables_completed, steps_completed, outcables_completed = self.advance_pipeline( - run_to_advance=curr_RS.child_run, - incables_completed=incables_completed, - steps_completed=steps_completed, - outcables_completed=outcables_completed - ) - - curr_RS.refresh_from_db() - if curr_RS.child_run.is_cancelled(): - curr_RS.cancel_running(save=True) - run_to_resume.cancel(save=True) - return incables_completed, steps_completed, outcables_completed - elif curr_RS.child_run.is_failed(): - curr_RS.finish_failure(save=True) - run_to_resume.mark_failure(save=True) - return incables_completed, steps_completed, outcables_completed - elif curr_RS.child_run.is_successful(): - curr_RS.finish_successfully(save=True) - else: - all_complete = False - - # We've done all we can with this sub-Pipeline, so we move on to the next step. - continue - - # Now, check that this step is still pending. If not, skip ahead. - if not curr_RS.is_pending(): - continue - - # If this step is not fed at all by any of the tasks that just completed, - # we skip it -- it can't have just become ready to go. - # Special case: this step has no inputs (for example, it's a random number generator). - # If so, we just go ahead. - fed_by_newly_completed = not step.cables_in.exists() - if step.cables_in.filter(source_step__in=step_nums_completed).exists(): - fed_by_newly_completed = True - if not fed_by_newly_completed: - for cable in outcables_completed: - parent_runstep = cable.parent_run.parent_runstep - if parent_runstep is None or parent_runstep.run != run_to_resume: - continue - output_fed = parent_runstep.transformation.outputs.get( - dataset_idx=cable.pipelineoutputcable.output_idx - ) - if step.cables_in.filter(source_step=parent_runstep.step_num, source=output_fed).exists(): - fed_by_newly_completed = True - break - - if not fed_by_newly_completed and run_to_resume.is_subrun: - # Check if this is fed by a completed incable (i.e. if this is part of a sub-Pipeline that is - # fed directly from the inputs). - - pipeline_inputs_fed = [] - for incable in incables_completed: - if run_to_resume.parent_runstep != incable.dest_runstep: - continue - pipeline_inputs_fed.append(incable.PSIC.dest) - - are_any_used = step.cables_in.filter(source_step=0, source__in=pipeline_inputs_fed).exists() - if are_any_used: - fed_by_newly_completed = True - - if not fed_by_newly_completed: - # This one certainly isn't getting completed now. - all_complete = False - continue - - # Examine this step and see if all of the inputs are (at least symbolically) available. - step_inputs = [] - - # For each PSIC leading to this step, check if its required dataset is in the maps. - all_inputs_fed = True - for psic in step.cables_in.order_by("dest__dataset_idx"): - socket = psic.source.definite - run_to_query = run_to_resume - - # If the PSIC comes from another step, the generator is the source pipeline step, - # or the output cable if it's a sub-pipeline. - if psic.source_step != 0: - generator = pipeline_to_resume.steps.get(step_num=psic.source_step) - if socket.transformation.is_pipeline(): - run_to_query = run_to_resume.runsteps.get(pipelinestep=generator).child_run - generator = generator.transformation.pipeline.outcables.get(output_idx=socket.dataset_idx) - - # Otherwise, the psic comes from step 0. - else: - # If this step is not a subpipeline, the dataset was uploaded. - generator = None - # If this step is a subpipeline, then the run we are interested in is the parent run. - # Get the run and cable that feeds this PSIC. - if run_to_resume.parent_runstep is not None: - run_to_query = run_to_resume.parent_runstep.run - cables_into_subpipeline = run_to_resume.parent_runstep.pipelinestep.cables_in - generator = cables_into_subpipeline.get(dest=psic.source) - - if (run_to_query, generator, socket) in self.socket_map: - step_inputs.append(self.socket_map[(run_to_query, generator, socket)]) - else: - all_inputs_fed = False - break - - if not all_inputs_fed: - # This step cannot be run yet, so we move on. - all_complete = False - continue - - # Start execution of this step. - curr_run_coords = run_to_resume.get_coordinates() - curr_run_plan = self.run_plan - for coord in curr_run_coords: - curr_run_plan = curr_run_plan.step_plans[coord-1].subrun_plan - assert curr_RS == curr_run_plan.step_plans[step.step_num-1].run_step - run_dir = os.path.join(sandbox_path, "step{}".format(step.step_num)) - - step_coords = curr_RS.get_coordinates() - step_coord_render = step_coords if len(step_coords) > 1 else step_coords[0] - self.logger.debug("Beginning execution of step %s (%s)", step_coord_render, step) - - # At this point we know that all inputs are available. - if step.is_subpipeline(): - self.logger.debug("Step %s (coordinates %s) is a sub-Pipeline. Marking it " - "as started, and handling its input cables.", - curr_RS, curr_RS.get_coordinates()) - curr_RS.start(save=True) # transition: Pending->Running - _in_dir, _out_dir, log_dir = self._setup_step_paths(run_dir, False) - - # We start all of the RunSICs in motion. If they all successfully reuse, - # we advance the sub-pipeline. - self.sub_pipeline_cable_tracker[curr_RS] = set() - - all_RSICs_done = True - cable_info_list = [] - for input_cable in step.cables_in.order_by("dest__dataset_idx"): - - cable_path = self.step_xput_path(curr_RS, input_cable.dest, run_dir) - - cable_exec_info = self.reuse_or_prepare_cable( - input_cable, - curr_RS, - step_inputs[input_cable.dest.dataset_idx-1], - cable_path, - log_dir, - run_dir - ) - cable_info_list.append(cable_exec_info) - - cable_record = cable_exec_info.cable_record - if cable_record.is_complete(): - incables_completed.append(cable_record) - elif cable_exec_info.could_be_reused: - cable_record.finish_successfully() - incables_completed.append(cable_record) - else: - self.sub_pipeline_cable_tracker[curr_RS].add(cable_record) - all_RSICs_done = False - - # If the cable was cancelled (e.g. due to bad input), we bail. - return_because_fail = False - if cable_exec_info.cancelled: - self.logger.debug("Input cable %s to sub-pipeline step %s was cancelled", - cable_exec_info.cable_record, - curr_RS) - return_because_fail = True - elif (cable_exec_info.cable_record.reused and - not cable_exec_info.could_be_reused): - self.logger.debug("Input cable %s to sub-pipeline step %s failed on reuse", - cable_exec_info.cable_record, - curr_RS) - return_because_fail = True - - if return_because_fail: - curr_RS.refresh_from_db() - curr_RS.child_run.cancel() - curr_RS.child_run.stop(save=True) - - if cable_exec_info.cancelled: - curr_RS.cancel_running(save=True) - run_to_resume.cancel(save=True) - else: - curr_RS.finish_failure(save=True) - run_to_resume.mark_failure(save=True) - curr_RS.complete_clean() - - # We don't mark the Run as complete in case something is still running. - return incables_completed, steps_completed, outcables_completed - - # Bundle up the information required to process this step. - # Construct output_paths. - output_paths = [self.step_xput_path(curr_RS, x, run_dir) for x in step.outputs] - execute_info = RunStepExecuteInfo(curr_RS, self.user, cable_info_list, None, run_dir, log_dir, - output_paths) - self.step_execute_info[(run_to_resume, step)] = execute_info - - if not all_RSICs_done: - all_complete = False - else: - incables_completed, steps_completed, outcables_completed = self.advance_pipeline( - run_to_advance=curr_RS.child_run, - incables_completed=incables_completed, - steps_completed=steps_completed, - outcables_completed=outcables_completed - ) - - # Update states for curr_RS and run_to_resume if necessary. - curr_RS.refresh_from_db() - if curr_RS.child_run.is_cancelled(): - curr_RS.cancel_running(save=True) - run_to_resume.cancel(save=True) - return incables_completed, steps_completed, outcables_completed - elif curr_RS.child_run.is_failed(): - curr_RS.finish_failure(save=True) - run_to_resume.mark_failure(save=True) - return incables_completed, steps_completed, outcables_completed - elif curr_RS.child_run.is_successful(): - curr_RS.finish_successfully(save=True) - else: - # Nothing wrong, but the step is not finished. - all_complete = False - - # We've done all we can for this step. - continue - - # At this point, we know the step is not a sub-Pipeline, so we go about our business. - curr_RS = self.reuse_or_prepare_step(step, run_to_resume, step_inputs, run_dir) - - # If the step we just started is for a Method, and it was successfully reused, then we add its step - # number to the list of those just completed. This may then allow subsequent steps to also be started. - if curr_RS.is_cancelled(): - # If the RunStep is cancelled after reuse, that means that one of - # its input cables failed on reuse, or a cable cancelled because it - # was unable to copy a file into the sandbox. - failed_cables = curr_RS.RSICs.filter(_runcomponentstate__pk=runcomponentstates.FAILED_PK) - cancelled_cables = curr_RS.RSICs.filter(_runcomponentstate__pk=runcomponentstates.CANCELLED_PK) - assert failed_cables.exists() or cancelled_cables.exists() - - if failed_cables.exists(): - self.logger.debug("Input cable(s) %s to step %d (%s) failed", - failed_cables, step.step_num, step) - run_to_resume.mark_failure(save=True) - if cancelled_cables.exists(): - self.logger.debug("Input cable(s) %s to step %d (%s) cancelled", - cancelled_cables, step.step_num, step) - if not failed_cables.exists(): - run_to_resume.cancel(save=True) - - return incables_completed, steps_completed, outcables_completed - - elif curr_RS.reused and not curr_RS.is_successful(): - self.logger.debug("Step %d (%s) failed on reuse", step.step_num, step) - run_to_resume.mark_failure(save=True) - return incables_completed, steps_completed, outcables_completed - - elif curr_RS.is_successful(): - step_nums_completed.append(step.step_num) - - elif curr_RS.is_running(): - all_complete = False - - else: - # FIXME check that this is all the possible states that it can be in after r_o_p_s - raise RuntimeError("unhandled case !\n") - - # Now go through the output cables and do the same. - for outcable in pipeline_to_resume.outcables.order_by("output_idx"): - curr_cable = run_to_resume.runoutputcables.filter(pipelineoutputcable=outcable).first() - - # First, if this is already running or complete, we skip it. - if curr_cable is not None and not curr_cable.is_pending(): - if curr_cable.is_running(): - all_complete = False - continue - - # At this point we know this cable has not already been run; i.e. either - # there is no record or the record is still pending. - - # Check if this cable has just had its input made available. First, check steps that - # just finished. - source_dataset = None - fed_by_newly_completed = False - - feeder_pipeline_step = pipeline_to_resume.steps.get(step_num=outcable.source_step) - if outcable.source_step in step_nums_completed: - source_dataset = self.socket_map[( - run_to_resume, - feeder_pipeline_step, - outcable.source - )] - fed_by_newly_completed = True - - # Next, check outcables of sub-pipelines to see if they provide what we need. - if not fed_by_newly_completed: - for cable in outcables_completed: - parent_runstep = cable.parent_run.parent_runstep - if parent_runstep is None or parent_runstep.run != run_to_resume: - continue - output_fed = parent_runstep.transformation.outputs.get( - dataset_idx=cable.pipelineoutputcable.output_idx - ) - if outcable.source_step == parent_runstep.step_num and outcable.source == output_fed: - source_dataset = self.socket_map[(cable.parent_run, cable.pipelineoutputcable, output_fed)] - fed_by_newly_completed = True - break - - if not fed_by_newly_completed: - # This outcable cannot be run yet. - all_complete = False - continue - - # We can now start this cable. - file_suffix = "raw" if outcable.is_raw() else "csv" - out_file_name = "run{}_{}.{}".format(run_to_resume.pk, outcable.output_name, file_suffix) - output_path = os.path.join(self.out_dir, out_file_name) - source_step_execution_info = self.step_execute_info[(run_to_resume, feeder_pipeline_step)] - cable_exec_info = self.reuse_or_prepare_cable( - outcable, - run_to_resume, - source_dataset, - output_path, - source_step_execution_info.log_dir, - source_step_execution_info.step_run_dir - ) - - cr = cable_exec_info.cable_record - - if cr.is_failed(): - self.logger.debug("Cable %s failed on reuse", cr.pipelineoutputcable) - run_to_resume.mark_failure(save=True) - return incables_completed, steps_completed, outcables_completed - - elif cable_exec_info.could_be_reused: - cr.finish_successfully(save=True) - outcables_completed.append(cr) - - elif cr.is_running(): - all_complete = False - - # FIXME check that this is all the possible conditions coming out of r_o_p_c - - if all_complete and not run_to_resume.is_complete(): - self.logger.debug("Run (coordinates %s) completed.", run_to_resume.get_coordinates()) - with transaction.atomic(): - run_to_resume.stop(save=True) # this transitions the state appropriately. - - return incables_completed, steps_completed, outcables_completed - - def reuse_or_prepare_cable(self, cable, parent_record, input_dataset, output_path, - log_dir, cable_info_dir): - """ - Attempt to reuse the cable; prepare it for finishing if unable. - """ - assert input_dataset in self.dataset_fs_map - - # Create new RSIC/ROC. - curr_record = RunCable.create(cable, parent_record) # this start()s it - self.logger.debug("Not recovering - created {}".format(curr_record.__class__.__name__)) - self.logger.debug("Cable keeps output? {}".format(curr_record.keeps_output())) - - by_step = parent_record if isinstance(parent_record, RunStep) else None - - # We bail out if the input has somehow been corrupted. - if not input_dataset.initially_OK(): - # FIXME this should never happen because if it's an input, it will have been - # checked by Sandbox, and if it's a cable inside the Pipeline, whatever fed - # this cable should have failed. - self.logger.debug("Input %s failed its initial check and should not be used. Cancelling.", - input_dataset) - - # Update state variables. - curr_record.cancel_running(save=True) - curr_record.complete_clean() - - # Return a RunCableExecuteInfo that is marked as cancelled. - exec_info = RunCableExecuteInfo(curr_record, - self.user, - None, - input_dataset, - self.dataset_fs_map[input_dataset], - output_path, - log_dir=log_dir, - by_step=by_step) - exec_info.cancel() - self.cable_execute_info[(curr_record.parent_run, cable)] = exec_info - return exec_info - - # Attempt to reuse this PipelineCable. - return_now = False - - succeeded_yet = False - self.logger.debug("Checking whether cable can be reused") - could_be_reused = False - while not succeeded_yet: - try: - with transaction.atomic(): - curr_ER, can_reuse = curr_record.get_suitable_ER(input_dataset, reuse_failures=False) - - if curr_ER is not None: - output_dataset = curr_ER.execrecordouts.first().dataset - - if curr_ER.generator.record.is_quarantined(): - # We will re-attempt; if it's fixed, then we un-quarantine the ExecRecord. - self.logger.debug( - "Found quarantined ExecRecord %s; will decontaminate if successful", - curr_ER - ) - - elif not can_reuse["successful"] or can_reuse["fully reusable"]: - # In this case, we can return now (either successfully or not). - self.logger.debug( - "ExecRecord %s is reusable (successful = %s)", - curr_ER, - can_reuse["successful"] - ) - curr_record.reused = True - curr_record.execrecord = curr_ER - - if can_reuse["successful"]: # and therefore fully reusable - # curr_record.finish_successfully(save=True) - could_be_reused = True - else: - curr_record.finish_failure(save=True) - # curr_record.complete_clean() - - self.update_cable_maps(curr_record, output_dataset, output_path) - return_now = True - succeeded_yet = True - except (OperationalError, InternalError): - wait_time = random.random() - self.logger.debug("Database conflict. Waiting for %f seconds before retrying.", wait_time) - time.sleep(wait_time) - - # Bundle up execution info in case this needs to be run, either by recovery, as a first execution, - # or as a "filling-in". - exec_info = RunCableExecuteInfo(curr_record, - self.user, - curr_ER, - input_dataset, - self.dataset_fs_map[input_dataset], - output_path, - log_dir=log_dir, - by_step=by_step, - could_be_reused=could_be_reused) - - exec_info.set_cable_info_dir(cable_info_dir) - - self.cable_execute_info[(curr_record.parent_run, cable)] = exec_info - - if return_now: - curr_record.save() - return exec_info - - # We didn't find a compatible and reusable ExecRecord, so we are committed to executing - # this cable. - curr_record.reused = False - self.logger.debug("No ER to completely reuse - preparing execution of this cable") - - # Check the availability of input_dataset; recover if necessary. Queue for execution - # if cable is an outcable or an incable that feeds a sub-Pipeline. - try: - exec_info.ready_to_go = self.enqueue_cable(exec_info, force=False) - except Sandbox.RunInputEmptyException: - # This should have been cancelled already. - curr_record.refresh_from_db() - assert curr_record.is_cancelled() - curr_record.complete_clean() - - # Mark exec_info as cancelled. - exec_info.cancel() - - return exec_info - - # We'd call this when we need to prepare a cable for recovery. This is essentially a "force" version of - # reuse_or_prepare_cable, where we don't attempt at all to reuse (we're past that point and we know we need - # to produce real data here). We call this function if a non-trivial cable produces data that's fed into - # another step, e.g. an outcable from a sub-pipeline. - def enqueue_cable(self, cable_info, force=False): - """ - Recursive helper for recover that handles recovery of a cable. - - RAISES: - If this encounters a Run input that cannot be recovered, it raises - Sandbox.RunInputEmptyException. - """ - # Unpack info from cable_info. - cable_record = cable_info.cable_record - input_dataset = cable_info.input_dataset - curr_ER = cable_info.execrecord - recovering_record = cable_info.recovering_record - by_step = cable_info.by_step - - # If this cable is already on the queue, we can return. - if cable_record in self.queue_for_processing and by_step is None: - self.logger.debug("Cable is already slated for recovery") - return cable_record - - ready_to_go = False - if force: - assert curr_ER is not None - dataset_path = self.find_dataset(input_dataset) - # Add this cable to the queue, unlike in reuse_or_prepare_cable. - # If this is independent of any step recovery, we add it to the queue; either by marking it as - # waiting for stuff that's going to recover, or by throwing it directly onto the list of tasks to - # perform. - queue_cable = (cable_record.component.is_outcable() or - cable_record.dest_runstep.pipelinestep.is_subpipeline() or - (force and by_step is None)) - file_access_start = timezone.now() - if dataset_path is None and not input_dataset.has_data(): - file_access_end = timezone.now() - - # Bail out here if the input dataset is a Pipeline input and cannot be recovered - # (at this point we are probably in the case where an external file was removed). - if (cable_record.component.is_incable() and - cable_record.top_level_run == cable_record.parent_run and - cable_record.component.definite.source_step == 0): - self.logger.debug("Cannot recover cable input: it is a Run input") - self.logger.debug("Cancelling cable with pk=%d.", cable_record.pk) - - with transaction.atomic(): - # Create a failed IntegrityCheckLog. - iic = IntegrityCheckLog( - dataset=input_dataset, - runcomponent=cable_record, - read_failed=True, - start_time=file_access_start, - end_time=file_access_end, - user=self.user - ) - iic.clean() - iic.save() - - cable_record.cancel_running(save=True) - # Making sure the invoking record (either cable_record or something else - # that's recovering) is handled by the calling function. - - raise Sandbox.RunInputEmptyException() - - self.logger.debug("Cable input requires non-trivial recovery") - self.queue_recovery(input_dataset, recovering_record=recovering_record) - - if queue_cable: - self.tasks_waiting[input_dataset].append(cable_record) - self.waiting_for[cable_record] = [input_dataset] - - else: - ready_to_go = True - if queue_cable: - self.queue_for_processing.append(cable_record) - - return ready_to_go - - # Function that reuses or prepares a step, which will later be complemented by a finish_step - # method. This would not be called if you were recovering. This will not be called - # on steps that are sub-Pipelines. - def reuse_or_prepare_step(self, pipelinestep, parent_run, inputs, step_run_dir): - """ - Reuse step if possible; prepare it for execution if not. - - As in execute_step: - Inputs written to: [step run dir]/input_data/step[step num]_[input name] - Outputs written to: [step run dir]/output_data/step[step num]_[output name] - Logs written to: [step run dir]/logs/step[step num]_std(out|err).txt - """ - - # Start execution of this step. - # Retrieve the RunStep from the run plan. - curr_run_coords = parent_run.get_coordinates() - curr_run_plan = self.run_plan - for coord in curr_run_coords: - curr_run_plan = curr_run_plan.step_plans[coord-1].subrun_plan - - step_plan = curr_run_plan.step_plans[pipelinestep.step_num-1] - curr_RS = step_plan.run_step - curr_RS.start() - - _in_dir, _out_dir, log_dir = self._setup_step_paths(step_run_dir, False) - - assert not pipelinestep.is_subpipeline() - - # Note: bad inputs will be caught by the cables. - input_names = ", ".join(str(i) for i in inputs) - self.logger.debug("Beginning execution of step {} in directory {} on inputs {}" - .format(pipelinestep, step_run_dir, input_names)) - - # Check which steps we're waiting on. - # Make a note of those steps that feed cables that are reused, but do not retain their output, - # i.e. those cables that are *symbolically* reusable but not *actually* reusable. - datasets_to_recover = [] - symbolically_okay_datasets = [] - cable_info_list = [] - for i, curr_input in enumerate(pipelinestep.inputs): # This is already ordered! - # The cable that feeds this input and where it will write its eventual output. - corresp_cable = pipelinestep.cables_in.get(dest=curr_input) - cable_path = self.step_xput_path(curr_RS, curr_input, step_run_dir) - cable_exec_info = self.reuse_or_prepare_cable( - corresp_cable, - curr_RS, - inputs[i], - cable_path, - log_dir, - step_run_dir - ) - cable_info_list.append(cable_exec_info) - - # If the cable was cancelled (e.g. due to bad input), we bail. - return_because_fail = False - if cable_exec_info.cable_record.is_cancelled(): - self.logger.debug("Input cable %s to step %s was cancelled", cable_exec_info.cable_record, - curr_RS) - return_because_fail = True - elif (cable_exec_info.cable_record.reused and - cable_exec_info.cable_record.is_failed()): - self.logger.debug("Input cable %s to step %s failed on reuse", cable_exec_info.cable_record, - curr_RS) - return_because_fail = True - # If the cable is not fit to be reused and not ready to go, we need to recover its input. - # elif not cable_exec_info.cable_record.is_complete(): - elif not cable_exec_info.could_be_reused: - if not cable_exec_info.ready_to_go: - datasets_to_recover.append(inputs[i]) - elif not cable_exec_info.execrecord.execrecordins.first().dataset.has_data(): - symbolically_okay_datasets.append(inputs[i]) - - if return_because_fail: - curr_RS.cancel_running(save=True) - curr_RS.complete_clean() - return curr_RS - - # Having reached here, we know that all input cables are either fit for reuse or ready to go. - - # Construct output_paths. - output_paths = [self.step_xput_path(curr_RS, x, step_run_dir) for x in pipelinestep.outputs] - execute_info = RunStepExecuteInfo( - curr_RS, - self.user, - cable_info_list, - None, - step_run_dir, - log_dir, - output_paths - ) - self.step_execute_info[(parent_run, pipelinestep)] = execute_info - - # If we're waiting on feeder steps, register this step as waiting for other steps, - # and return the (incomplete) RunStep. Note that if this happens, we will have to - # recover the feeders that were only symbolically OK, so we add those to the pile. - # The steps that need to precede it have already been added to the queue above by - # reuse_or_prepare_cable. - if len(datasets_to_recover) > 0: - for input_dataset in datasets_to_recover + symbolically_okay_datasets: - self.tasks_waiting[input_dataset].append(curr_RS) - self.waiting_for[curr_RS] = datasets_to_recover + symbolically_okay_datasets - return curr_RS - - # At this point we know that we're at least symbolically OK to proceed. - # Check if all of the cables have known outputs; if they do, then we can - # attempt to reuse an ExecRecord. - inputs_after_cable = [] - all_inputs_present = True - for i, curr_input in enumerate(pipelinestep.inputs): - if not cable_info_list[i].could_be_reused: - all_inputs_present = False - break - inputs_after_cable.append(cable_info_list[i].cable_record.execrecord.execrecordouts.first().dataset) - - if all_inputs_present: - - # Look for a reusable ExecRecord. If we find it, then complete the RunStep. - succeeded_yet = False - while not succeeded_yet: - try: - with transaction.atomic(): - curr_ER = step_plan.execrecord - can_reuse = curr_ER and curr_RS.check_ER_usable(curr_ER) - - if curr_ER is not None: - execute_info.execrecord = curr_ER - - if curr_ER.generator.record.is_quarantined(): - # We will re-attempt; if it's fixed, then we un-quarantine the ExecRecord. - self.logger.debug( - "Found quarantined ExecRecord %s; will decontaminate if successful", - curr_ER - ) - elif can_reuse["successful"] and not can_reuse["fully reusable"]: - self.logger.debug("Filling in ExecRecord {}".format(curr_ER)) - - else: - # This is either unsuccessful or fully reusable, so we can return. - self.logger.debug( - "ExecRecord {} is reusable (successful = {})".format( - curr_ER, can_reuse["successful"]) - ) - curr_RS.reused = True - curr_RS.execrecord = curr_ER - - for cable_info in cable_info_list: - # Being here means all cables were able to be fully reused. - cable_info.cable_record.finish_successfully(save=True) - - if can_reuse["successful"]: - curr_RS.finish_successfully(save=True) - else: - curr_RS.finish_failure(save=True) - - self.update_step_maps(curr_RS, step_run_dir, output_paths) - return curr_RS - - else: - self.logger.debug("No compatible ExecRecord found yet") - - curr_RS.reused = False - curr_RS.save() - succeeded_yet = True - except (OperationalError, InternalError): - wait_time = random.random() - self.logger.debug("Database conflict. Waiting for %f seconds before retrying.", wait_time) - time.sleep(wait_time) - - # We found no reusable ER, so we add this step to the queue. - # If there were any inputs that were only symbolically OK, we call queue_recover on them and register - # this step as waiting for them. - if len(symbolically_okay_datasets) > 0: - for missing_data in symbolically_okay_datasets: - try: - self.queue_recovery(missing_data, invoking_record=curr_RS) - except Sandbox.RunInputEmptyException: - self.logger.debug("Cancelling RunStep with pk=%d.", curr_RS.pk) - - # Update state variables. - curr_RS.cancel_running(save=True) - curr_RS.complete_clean() - execute_info.cancel() - return curr_RS - - self.tasks_waiting[missing_data].append(curr_RS) - self.waiting_for[curr_RS] = symbolically_okay_datasets - else: - # We're not waiting for any inputs. Add this step to the queue. - self.queue_for_processing.append(curr_RS) - - return curr_RS - - def step_recover_h(self, execute_info): - """ - Helper for recover that's responsible for forcing recovery of a step. - - RAISES: - Sandbox.RunInputEmptyException if it encounters an empty Run input. - """ - # Break out execute_info. - runstep = execute_info.runstep - cable_info_list = execute_info.cable_info_list - recovering_record = execute_info.recovering_record - - pipelinestep = runstep.pipelinestep - assert not pipelinestep.is_subpipeline() - - # If this runstep is already on the queue, we can return. - if runstep in self.queue_for_processing: - self.logger.debug("Step already in queue for execution") - return - - # Check which cables need to be re-run. Since we're forcing this step, we can't have - # cables which are only symbolically OK, we need their output to be available either in the - # sandbox or in the database. - datasets_to_recover_first = [] - for cable in cable_info_list: - # We use the sandbox's version of the execute information for this cable. - cable_info = self.cable_execute_info[(cable.cable_record.parent_run, cable.cable_record.PSIC)] - - # If the cable needs its feeding steps to recover, we throw them onto the queue if they're not there - # already. - cable_out_dataset = cable_info.execrecord.execrecordouts.first().dataset - if not cable_out_dataset.has_data() and not self.find_dataset(cable_out_dataset): - datasets_to_recover_first.append(cable_info.input_dataset) - execute_info.flag_for_recovery(recovering_record, by_step=runstep) - self.enqueue_cable(execute_info, force=True) - - if len(datasets_to_recover_first) > 0: - for dataset in datasets_to_recover_first: - self.tasks_waiting[dataset].append(runstep) - self.waiting_for[runstep] = datasets_to_recover_first - else: - self.queue_for_processing.append(runstep) - - def queue_recovery(self, dataset_to_recover, invoking_record): - """ - Determines and enqueues the steps necessary to reproduce the specified Dataset. - - This is an MPI-friendly version of recover. It only ever handles non-trivial recoveries, - as trivial recoveries are now performed by cables themselves. - - @param dataset_to_recover: dataset that needs to be recovered - @param invoking_record: the run component that needs the - dataset as an input - - RAISES: - Sandbox.RunInputEmptyException if it encounters an empty Run input. - - PRE - dataset_to_recover is in the maps but no corresponding file is on the file system. - """ - # NOTE before we recover an dataset, we should look to see if it's already being recovered - # by something else. If so we can just wait for that to finish. - assert dataset_to_recover in self.dataset_fs_map - assert not self.find_dataset(dataset_to_recover) - - self.logger.debug("Performing computation to create missing Dataset") - - # Search for the generator of the dataset in the Pipeline. - curr_run, generator = self.first_generator_of_dataset(dataset_to_recover) - - if curr_run is None: - raise ValueError('Dataset "{}" was not found in Pipeline "{}" and cannot be recovered' - .format(dataset_to_recover, self.pipeline)) - elif generator is None: - raise ValueError('Dataset "{}" is an input to Pipeline "{}" and cannot be recovered' - .format(dataset_to_recover, self.pipeline)) - - # We're now going to look up what we need to run from cable_execute_info and step_execute_info. - - self.logger.debug('Processing {} "{}" in recovery mode'.format(generator.__class__.__name__, generator)) - if type(generator) == pipeline.models.PipelineStep: - curr_execute_info = self.step_execute_info[(curr_run, generator)] - curr_execute_info.flag_for_recovery(invoking_record) - self.step_recover_h(curr_execute_info) - else: - # Look for this cable (may be PSIC or POC) in curr_execute_info. - curr_execute_info = self.cable_execute_info[(curr_run, generator)] - curr_execute_info.flag_for_recovery(invoking_record) - self.enqueue_cable(curr_execute_info, force=True) - - def hand_tasks_to_fleet(self): - ready_tasks = self.queue_for_processing - self.queue_for_processing = [] - return ready_tasks - - def get_task_info(self, task): - """ - Helper that retrieves the task information for the specified RunStep/RunCable. - """ - assert task.top_level_run == self.run - if type(task) == RunStep: - return self.step_execute_info[(task.run, task.pipelinestep)] - return self.cable_execute_info[(task.parent_run, task.component)] - - def update_sandbox(self, task_finished): - """ - Helper that updates the sandbox maps to reflect the information from the specified task_finished. - - PRE: task_finished is a RunStep/RunCable belonging to this sandbox's run, and it already has - execution info available in step_execute_info or cable_execute_info. - """ - assert task_finished.top_level_run == self.run - if type(task_finished) == RunStep: - assert (task_finished.run, task_finished.pipelinestep) in self.step_execute_info - else: - assert (task_finished.parent_run, task_finished.component) in self.cable_execute_info - - task_execute_info = self.get_task_info(task_finished) - - # Update the sandbox with this information. - if type(task_finished) == RunStep: - # Update the sandbox maps with the input cables' information as well as that - # of the step itself. - for rsic in task_finished.RSICs.all(): - curr_cable_task_info = self.get_task_info(rsic) - curr_cable_out_dataset = rsic.execrecord.execrecordouts.first().dataset - self.update_cable_maps(rsic, curr_cable_out_dataset, curr_cable_task_info.output_path) - - self.update_step_maps(task_finished, task_execute_info.step_run_dir, task_execute_info.output_paths) - - else: - cable_out_dataset = task_finished.execrecord.execrecordouts.first().dataset - self.update_cable_maps(task_finished, cable_out_dataset, task_execute_info.output_path) - - @staticmethod - def _setup_step_paths(step_run_dir, recover): - """Set up paths for running a PipelineStep. - - INPUTS - step_run_dir root directory where step will be run - recover are we recovering? (if so, test if the - directories are there instead of creating them) - - OUTPUTS - in_dir directory to put step's inputs - out_dir directory where step will put its outputs - log_dir directory to put logs in - """ - log_dir = os.path.join(step_run_dir, dirnames.LOG_DIR) - out_dir = os.path.join(step_run_dir, dirnames.OUT_DIR) - in_dir = os.path.join(step_run_dir, dirnames.IN_DIR) - - for workdir in [step_run_dir, log_dir, out_dir, in_dir]: - file_access_utils.set_up_directory(workdir, tolerate=recover) - file_access_utils.configure_sandbox_permissions(workdir) - return (in_dir, out_dir, log_dir) - - @staticmethod - def finish_cable(cable_execute_dict): - """ - Finishes an un-reused cable that has already been prepared for execution. - - This code is intended to be run as a Slurm task, so it's a static method that - takes a dictionary rather than a RunCableExecuteInfo object. - - If we are reaching this point, we know that the data required for input_dataset is either - in place in the sandbox or available in the database. - - This function is called by finish_step, because we want it to be called by the same - worker(s) as the step is, so its output is on the local filesystem of the worker, which - may be a remote MPI host. - """ - # Retrieve info from the database using the PKs passed. - curr_record = RunComponent.objects.get(pk=cable_execute_dict["cable_record_pk"]).definite - input_dataset = Dataset.objects.get(pk=cable_execute_dict["input_dataset_pk"]) - output_path = cable_execute_dict["output_path"] - recovering_record = None - if cable_execute_dict["recovering_record_pk"] is not None: - recovering_record = RunComponent.objects.get( - pk=cable_execute_dict["recovering_record_pk"] - ).definite - curr_ER = None - - make_dataset = curr_record.keeps_output() - dataset_name = curr_record.output_name() - dataset_desc = curr_record.output_description() - - recover = recovering_record is not None - invoking_record = recovering_record or curr_record - if recover: - curr_record.begin_recovery(save=True) - - # It's possible that this cable was completed in the time between the Manager queueing this task - # and the worker starting it. If so we can use the ExecRecord, and maybe even fully reuse it - # if this was not called by finish_step. - succeeded_yet = False - while not succeeded_yet: - try: - with transaction.atomic(): - if cable_execute_dict["execrecord_pk"] is not None: - curr_ER = ExecRecord.objects.get(pk=cable_execute_dict["execrecord_pk"]) - can_reuse = curr_record.check_ER_usable(curr_ER) - else: - curr_ER, can_reuse = curr_record.get_suitable_ER(input_dataset, reuse_failures=False) - - if curr_ER is not None: - if curr_ER.generator.record.is_quarantined(): - # We will re-attempt; if it's fixed, then we un-quarantine the ExecRecord. - logger.debug( - "Found quarantined ExecRecord %s; will decontaminate if successful", - curr_ER - ) - - # If it was unsuccessful, we bail. Alternately, if we can fully reuse it now and don't need to - # execute it for a parent step, we can return. - if (not can_reuse["successful"] or - (can_reuse["fully reusable"] and cable_execute_dict["by_step_pk"] is None)): - logger.debug("ExecRecord %s is reusable (successful = %s)", - curr_ER, can_reuse["successful"]) - curr_record.reused = True - curr_record.execrecord = curr_ER - - if can_reuse["successful"]: - # This is a fully-reusable record. - curr_record.finish_successfully(save=True) - else: - # Mark curr_record as failed; if this is a recovery, recovering_record - # will be handled elsewhere - curr_record.finish_failure(save=True) - - curr_record.complete_clean() - return curr_record - succeeded_yet = True - except (OperationalError, InternalError): - wait_time = random.random() - logger.debug("Database conflict. Waiting for %f seconds before retrying.", - wait_time) - time.sleep(wait_time) - - # We're now committed to actually running this cable. - input_dataset_path = cable_execute_dict["input_dataset_path"] - user = User.objects.get(pk=cable_execute_dict["user_pk"]) - - # Preconditions to test. - # assert curr_record is not None - input_dataset_in_sdbx = file_access_utils.file_exists(input_dataset_path) - - cable = curr_record.definite.component - - # Write the input dataset to the sandbox if necessary. - # FIXME at some point in the future this will have to be updated to mean "write to the local sandbox". - if not input_dataset_in_sdbx: - logger.debug("Dataset is in the DB - writing it to the file system") - file_path = None - if bool(input_dataset.dataset_file): - file_path = input_dataset.dataset_file.path - elif input_dataset.external_path: - file_path = input_dataset.external_absolute_path() - - copy_start = timezone.now() - fail_now = False - - try: - copy_and_confirm(file_path, input_dataset_path) - except (IOError, FileCreationError): - logger.error("Could not copy file %s to file %s.", - file_path, - input_dataset_path, - exc_info=True) - fail_now = True - finally: - copy_end = timezone.now() - - if fail_now: - with transaction.atomic(): - # Create a failed IntegrityCheckLog. - iic = IntegrityCheckLog( - dataset=input_dataset, - runcomponent=curr_record, - read_failed=True, - start_time=copy_start, - end_time=copy_end, - user=user - ) - iic.clean() - iic.save() - - else: - # Perform an integrity check since we've just copied this file to the sandbox for the - # first time. - logger.debug("Checking file just copied to sandbox for integrity.") - check = input_dataset.check_integrity( - input_dataset_path, - user, - execlog=None, - runcomponent=curr_record - ) - fail_now = check.is_fail() - - # Check again in case it failed in the previous else block. - if fail_now: - curr_record.cancel_running(save=True) - return curr_record - - if not recover: - # Get or create CDT for cable output (Evaluate cable wiring) - output_CDT = input_dataset.get_cdt() - if not cable.is_trivial(): - output_CDT = cable.find_compounddatatype() or cable.create_compounddatatype() - - else: - logger.debug("Recovering - will update old ER") - output_dataset = curr_ER.execrecordouts.first().dataset - output_CDT = output_dataset.get_cdt() - - curr_log = ExecLog.create(curr_record, invoking_record) - - # Run cable (this completes the ExecLog). - cable_failed = False - file_size_unstable = False - try: - md5 = cable.run_cable(input_dataset_path, output_path, curr_record, curr_log) - except (OSError, FileCreationError) as e: - cable_failed = True - logger.error("could not run cable %s to file %s.", - input_dataset_path, - output_path, - exc_info=True) - if hasattr(e, "md5"): - # Cable failed on running so there's an MD5 attached. - md5 = e.md5 - file_size_unstable = True - - # Here, we're authoring/modifying an ExecRecord, so we use a transaction. - preexisting_ER = curr_ER is not None - succeeded_yet = False - while not succeeded_yet: - try: - with transaction.atomic(): - bad_output = False - start_time = timezone.now() - if cable_failed: - end_time = timezone.now() - bad_output = True - - # It's conceivable that the linking could fail in the - # trivial case; in which case we should associate a "missing data" - # check to input_dataset == output_dataset. - if cable.is_trivial(): - output_dataset = input_dataset - elif curr_ER is None: - if not file_size_unstable: - output_dataset = Dataset.create_empty( - cdt=output_CDT, - file_source=curr_record - ) - else: - output_dataset = Dataset.create_dataset( - output_path, - cdt=output_CDT, - keep_file=make_dataset, - name=dataset_name, - description=dataset_desc, - file_source=curr_record, - check=False, - precomputed_md5=md5 - ) - else: - output_dataset = curr_ER.execrecordouts.first().dataset - - if not file_size_unstable: - output_dataset.mark_missing(start_time, end_time, curr_log, user) - else: - output_dataset.mark_file_not_stable(start_time, end_time, curr_log, user) - - # Update state variables. - curr_record.finish_failure(save=True) - if preexisting_ER: - curr_ER.quarantine_runcomponents() - - elif cable.is_trivial(): - output_dataset = input_dataset - - else: - # Do we need to keep this output? - if not make_dataset: - logger.debug("Cable doesn't keep output: not creating a dataset") - - if curr_ER is not None: - output_dataset = curr_ER.execrecordouts.first().dataset - if make_dataset: - output_dataset.register_file(output_path) - - else: - output_dataset = Dataset.create_dataset( - output_path, - cdt=output_CDT, - keep_file=make_dataset, - name=dataset_name, - description=dataset_desc, - file_source=curr_record, - check=False, - precomputed_md5=md5 - ) - - # Link the ExecRecord to curr_record if necessary, creating it if necessary also. - if not recover: - if curr_ER is None: - logger.debug("No ExecRecord already in use - creating fresh cable ExecRecord") - # Make ExecRecord, linking it to the ExecLog. - curr_ER = ExecRecord.create( - curr_log, - cable, - [input_dataset], - [output_dataset] - ) - # Link ER to RunCable (this may have already been linked; that's fine). - curr_record.link_execrecord(curr_ER, reused=False) - - else: - logger.debug("This was a recovery - not linking RSIC/RunOutputCable to ExecRecord") - succeeded_yet = True - except (OperationalError, InternalError): - wait_time = random.random() - logger.debug("Database conflict. Waiting for %f seconds before retrying.", wait_time) - time.sleep(wait_time) - - #### - # Check outputs - #### - if not bad_output: - # Case 1: the cable is trivial. Don't check the integrity, it was already checked - # when it was first written to the sandbox. - if cable.is_trivial(): - logger.debug("Cable is trivial; skipping integrity check") - - else: - # Case 2a: ExecRecord already existed and its output had been properly vetted. - # Case 2b: this was a recovery. - # Check the integrity of the output. - if ((preexisting_ER and (output_dataset.is_OK() or - output_dataset.any_failed_checks())) or - cable.is_trivial() or recover): - logger.debug("Performing integrity check of trivial or previously generated output") - # Perform integrity check. Note: if this fails, it will notify all RunComponents using it. - check = output_dataset.check_integrity(output_path, - user, - curr_log, - newly_computed_MD5=md5) - - # Case 3: the Dataset, one way or another, is not properly vetted. - else: - logger.debug("Output has no complete content check; performing content check") - summary_path = "{}_summary".format(output_path) - # Perform content check. Note: if this fails, it will notify all RunComponents using it. - check = output_dataset.check_file_contents( - output_path, - summary_path, - cable.min_rows_out, - cable.max_rows_out, - curr_log, - user - ) - - if check.is_fail(): - curr_record.finish_failure(save=True) - - logger.debug("DONE EXECUTING %s '%s'", type(cable).__name__, cable) - - # End. Return curr_record. Update state if necessary (i.e. if it hasn't already failed). - if curr_record.is_running(): - curr_record.finish_successfully(save=True) - curr_record.complete_clean() - return curr_record - - @staticmethod - def step_execution_setup(step_execute_dict): - """ - Prepare to carry out the task specified by step_execute_dict. - - This is intended to be run as a Slurm job, so is a static method. - I.e. in fleet.workers.Managersubmit_runstep(): - a dict containing pertinent information about the required run is written into a JSON file. - Then, using sbatch, a slurm job that calls 'manage.py step_helper' is submitted. - Upon execution by slurm, that command retrieves the dict from the json file and calls - this static method with the dict as an argument. - - Precondition: the task must be ready to go, i.e. its inputs must all be in place. Also - it should not have been run previously. This should not be a RunStep representing a Pipeline. - """ - # Break out the execution info. - curr_exec_rec = None - preexisting_exec_rec = step_execute_dict["execrecord_pk"] is not None - cable_info_dicts = step_execute_dict["cable_info_dicts"] - step_run_dir = step_execute_dict["step_run_dir"] - - recovering_record = None - if step_execute_dict["recovering_record_pk"] is not None: - # noinspection PyUnresolvedReferences - recovering_record = RunComponent.objects.get( - pk=step_execute_dict["recovering_record_pk"] - ).definite - - # already start()ed - curr_runstep = RunStep.objects.get(pk=step_execute_dict["runstep_pk"]) - assert not curr_runstep.pipelinestep.is_subpipeline() - method = curr_runstep.pipelinestep.transformation.definite - - try: - recover = recovering_record is not None - invoking_record = recovering_record or curr_runstep - - if recover: - curr_runstep.begin_recovery(save=True) - - #### - # Gather inputs: finish all input cables -- we want them written to the sandbox now, which is never - # done by reuse_or_prepare_cable. - input_paths = [] - completed_cable_pks = [] - for curr_execute_dict in cable_info_dicts: - # Update the cable execution information with the recovering record if applicable. - if recover: - curr_execute_dict["recovering_record_pk"] = recovering_record.pk - - curr_cable = Sandbox.finish_cable(curr_execute_dict) - completed_cable_pks.append(curr_cable.pk) - - # Cable failed, return incomplete RunStep. - curr_cable.refresh_from_db() - if not curr_cable.is_successful(): - logger.error("PipelineStepInputCable %s %s.", curr_cable, curr_cable.get_state_name()) - - # Cancel the other RunSICs for this step. - for rsic in curr_runstep.RSICs.exclude(pk__in=completed_cable_pks): - rsic.cancel_pending(save=True) - - # Update state variables. - curr_runstep.refresh_from_db() - curr_runstep.cancel_running(save=True) # Transition: Running->Cancelled - curr_runstep.complete_clean() - return curr_runstep - - # Cable succeeded. - input_paths.append(curr_execute_dict["output_path"]) - - # Check again to see if the ExecRecord was completed while this task - # waited on the queue. If this isn't a recovery, we can just stop. - if recover: - assert preexisting_exec_rec - curr_exec_rec = ExecRecord.objects.get(pk=step_execute_dict["execrecord_pk"]) - else: - curr_runstep.reused = False - curr_runstep.save() - - pipelinestep = curr_runstep.pipelinestep - - # Run code, creating ExecLog and MethodOutput. - curr_log = ExecLog.create(curr_runstep, invoking_record) - - # Check the integrity of the code before we run. - if not pipelinestep.transformation.definite.check_md5(): # this checks the driver and dependencies - logger.error("Method code has gone corrupt for %s or its " - "dependencies; stopping step", - pipelinestep.transformation.definite.driver) - # Stop everything! - curr_log.start() - curr_log.methodoutput.are_checksums_OK = False - curr_log.methodoutput.save() - curr_log.stop(save=True, clean=False) - - # Update state variables: - curr_runstep.finish_failure(save=True) - curr_runstep.complete_clean() - - logger.debug("Quarantining any other RunComponents using the same ExecRecord") - if preexisting_exec_rec: - curr_exec_rec.quarantine_runcomponents() # this is transaction'd - - # Install the code. - try: - method.install(step_run_dir) - except (IOError, FileCreationError): - logger.error("Method code failed to install; stopping step.", - exc_info=True) - curr_log.start() - curr_log.methodoutput.install_failed = True - curr_log.methodoutput.save() - curr_log.stop(save=True, clean=False) - curr_runstep.finish_failure(save=True) - curr_runstep.complete_clean() - - except KeyboardInterrupt: - curr_runstep.cancel_running(save=True) - raise StopExecution( - "Execution of step {} (method {}) was stopped during setup.".format( - curr_runstep, - method - ) - ) - - return curr_runstep - - def submit_step_execution(self, step_execute_info, after_okay, - slurm_sched_class, - docker_handler_class, - singularity_handler_class=None): - """ - Submit the step execution to Slurm. - - step_execute_info is an object of class RunStepExecuteInfo. - after_okay is a list of Slurm job handles; These are submitted jobs that must be - completed (successfully) before the execution of this driver can proceed. - - NOTE: when submitting the driver to slurm, the jobscript to run must be wrapped. - This is because at the time of submission, the driver code has not been installed yet, - and slurm submission will fail. - The way of getting around this problem is the following: - a) at slurm submission time: - create a small shell script wrapper which calls the not-yet existing driver. - b) at slurm run time: - the setup script, which must run successfully before this one is started, - has copied the driver into place, which the wrapper code now can run. - - NOTE: for docker support, we wrap the wrapper script again in order to launch it - within a docker container. - """ - # From here on the code is assumed not to be corrupted, and all the required files - # have been placed in their right places. - curr_run_step = step_execute_info.runstep - - input_paths = [x.output_path for x in step_execute_info.cable_info_list] - method = curr_run_step.pipelinestep.transformation.definite - dependencies = method.dependencies - dependency_paths = [os.path.join(dep.path, dep.get_filename()) - for dep in dependencies.all()] - # Driver name - docker_image = method.docker_image - image_id = (docker_image and docker_image.hash) - container_id = method.container_id - container = None - if container_id is None and image_id is not None: - handler_class = docker_handler_class - container_file = None - else: - handler_class = singularity_handler_class - if container_id is None: - container_file = os.path.join(settings.MEDIA_ROOT, - Container.UPLOAD_DIR, - settings.DEFAULT_CONTAINER) - else: - container = Container.objects.get(id=container_id) - container_file = container.get_absolute_path() - driver = method.driver - if driver is None: - driver_name = (container and container.family.name or - docker_image and docker_image.full_name) - driver_filename = None - else: - driver_name = driver_filename = driver.coderesource.filename - - coordinates = curr_run_step.get_coordinates() - if len(coordinates) == 1: - coord_str = coordinates[0] - else: - coord_str = "({})".format(",".join(str(x) for x in coordinates)) - job_name = "r{}s{}driver[{}]".format( - curr_run_step.top_level_run.pk, - coord_str, - driver_name) - logger.debug("Submitting driver '%s', task_pk %d", driver_filename, curr_run_step.pk) - # Collect information we need for the wrapper script - host_rundir = step_execute_info.step_run_dir - launch_args = handler_class.generate_launch_args( - host_rundir, - input_paths, - step_execute_info.output_paths, - driver_filename, - dependency_paths, - image_id=image_id, - container_file=container_file) - job_handle = slurm_sched_class.submit_job( - host_rundir, - launch_args[0], - launch_args[1:], - self.run.priority, - step_execute_info.threads_required, - step_execute_info.driver_stdout_path(), - step_execute_info.driver_stderr_path(), - after_okay=after_okay, - job_name=job_name, - mem=method.memory - ) - - return job_handle - - @staticmethod - def step_execution_bookkeeping(step_execute_dict): - """ - Perform bookkeeping after step execution. - - This is intended to run as a Slurm task, so it's coded as a static method. - """ - # Break out step_execute_dict. - curr_run_step = RunStep.objects.get(pk=step_execute_dict["runstep_pk"]) - curr_log = curr_run_step.log - curr_exec_rec = ( - None - if step_execute_dict["execrecord_pk"] is None - else ExecRecord.objects.get(pk=step_execute_dict["execrecord_pk"])) - pipelinestep = curr_run_step.pipelinestep - output_paths = step_execute_dict["output_paths"] - user = User.objects.get(pk=step_execute_dict["user_pk"]) - - recovering_record = None - if step_execute_dict["recovering_record_pk"] is not None: - # noinspection PyUnresolvedReferences - recovering_record = RunComponent.objects.get( - pk=step_execute_dict["recovering_record_pk"] - ).definite - recover = recovering_record is not None - - cable_info_dicts = step_execute_dict["cable_info_dicts"] - inputs_after_cable = [] - for curr_execute_dict in cable_info_dicts: - # noinspection PyUnresolvedReferences - curr_cable = RunSIC.objects.get(pk=curr_execute_dict["cable_record_pk"]) - inputs_after_cable.append(curr_cable.execrecord.execrecordouts.first().dataset) - - bad_execution = False - bad_output_found = False - integrity_checks = {} # {output_idx: check} - md5s = {} # {output_idx: md5} - try: - succeeded_yet = False - while not succeeded_yet: - try: - with transaction.atomic(): - # Create outputs. - # bad_output_found indicates we have detected problems with the output. - bad_output_found = False - bad_execution = not curr_log.is_successful() - output_datasets = [] - logger.debug("ExecLog.is_successful() == %s", not bad_execution) - - # if not recover: - if curr_exec_rec is not None: - if not recover: - logger.debug("Filling in pre-existing ExecRecord with PipelineStep outputs") - else: - logger.debug("Examining outputs of pre-existing ExecRecord after recovery") - else: - logger.debug("Creating new Datasets for PipelineStep outputs") - - for i, curr_output in enumerate(pipelinestep.outputs): - output_path = output_paths[i] - output_type = curr_output.get_cdt() - dataset_name = curr_run_step.output_name(curr_output) - dataset_desc = curr_run_step.output_description(curr_output) - make_dataset = curr_run_step.keeps_output(curr_output) - - # Check that the file exists, as we did for cables. - start_time = timezone.now() - file_confirmed = True - - try: - md5s[i] = file_access_utils.confirm_file_created(output_path) - except FileCreationError as e: - logger.warn("File at %s was not properly created.", output_path, exc_info=True) - file_confirmed = False - - if hasattr(e, "md5"): - md5s[i] = e.md5 - else: - md5s[i] = None - - if not file_confirmed: - end_time = timezone.now() - bad_output_found = True - - if curr_exec_rec is not None: - output_dataset = curr_exec_rec.get_execrecordout(curr_output).dataset - if md5s[i] is None: - output_dataset.mark_missing(start_time, end_time, curr_log, user) - else: - output_dataset.mark_file_not_stable(start_time, end_time, curr_log, user) - - elif md5s[i] is None: - output_dataset = Dataset.create_empty( - cdt=output_type, - file_source=curr_run_step - ) - output_dataset.mark_missing(start_time, end_time, curr_log, user) - else: - output_dataset = Dataset.create_dataset( - output_path, - cdt=output_type, - keep_file=make_dataset, - name=dataset_name, - description=dataset_desc, - file_source=curr_run_step, - check=False, - precomputed_md5=md5s[i] - ) - output_dataset.mark_file_not_stable(start_time, end_time, curr_log, user) - - else: - # If necessary, create new Dataset for output, and create the Dataset - # if it's to be retained. - if curr_exec_rec is not None: - output_ero = curr_exec_rec.get_execrecordout(curr_output) - if not make_dataset: - output_dataset = output_ero.dataset - else: - # Wrap in a transaction to prevent - # concurrent authoring of Datasets to - # an existing Dataset. - with transaction.atomic(): - output_dataset = Dataset.objects.select_for_update().filter( - pk=output_ero.dataset.pk).first() - if not output_dataset.has_data(): - check = output_dataset.check_integrity( - output_path, - checking_user=user, - execlog=curr_log, - notify_all=True, - newly_computed_MD5=md5s[i] - ) - integrity_checks[i] = check - if not check.is_fail(): - output_dataset.register_file(output_path) - - else: - output_dataset = Dataset.create_dataset( - output_path, - cdt=output_type, - keep_file=make_dataset, - name=dataset_name, - description=dataset_desc, - file_source=curr_run_step, - check=False, - precomputed_md5=md5s[i] - ) - logger.debug("First time seeing file: saved md5 %s", - output_dataset.MD5_checksum) - - output_datasets.append(output_dataset) - - # Create ExecRecord if there isn't already one. - if curr_exec_rec is None: - # Make new ExecRecord, linking it to the ExecLog - logger.debug("Creating fresh ExecRecord") - curr_exec_rec = ExecRecord.create( - curr_log, - pipelinestep, - inputs_after_cable, - output_datasets - ) - - # Link ExecRecord to RunStep (it may already have been linked; that's fine). - # It's possible that reused may be either True or False (e.g. this is a recovery). - curr_run_step.link_execrecord(curr_exec_rec, curr_run_step.reused) - - succeeded_yet = True - except (OperationalError, InternalError): - wait_time = random.random() - logger.debug( - "Database conflict. Waiting for %f seconds before retrying.", - wait_time - ) - time.sleep(wait_time) - - # Having confirmed their existence, we can now perform proper integrity/content - # checks on the outputs. - for i, curr_output in enumerate(pipelinestep.outputs): - output_path = output_paths[i] - output_dataset = curr_exec_rec.get_execrecordout(curr_output).dataset - check = None - - if bad_execution: - logger.debug("Execution was unsuccessful; no check on %s was done", output_path) - elif bad_output_found: - logger.debug("Bad output found; no check on %s was done", output_path) - - # Recovering or filling in old ER? Yes. - elif curr_exec_rec is not None: - # Perform integrity check. - logger.debug("Dataset has been computed before, checking integrity of %s", - output_dataset) - check = integrity_checks.get(i) - if check is None: - check = output_dataset.check_integrity(output_path, user, curr_log, - newly_computed_MD5=md5s[i]) - - # We may also need to perform a content check if there isn't a complete - # one already. - if not check.is_fail() and not output_dataset.content_checks.filter( - end_time__isnull=False).exists(): - logger.debug("Output has no complete content check; performing content check") - summary_path = "{}_summary".format(output_path) - check = output_dataset.check_file_contents( - output_path, - summary_path, - curr_output.get_min_row(), - curr_output.get_max_row(), - curr_log, - user - ) - - # Recovering or filling in old ER? No. - else: - # Perform content check. - logger.debug("%s is new data - performing content check", output_dataset) - summary_path = "{}_summary".format(output_path) - check = output_dataset.check_file_contents( - output_path, - summary_path, - curr_output.get_min_row(), - curr_output.get_max_row(), - curr_log, - user - ) - - # Check OK? No. - if check and check.is_fail(): - logger.warn("%s failed for %s", check.__class__.__name__, output_path) - bad_output_found = True - - # Check OK? Yes. - elif check: - logger.debug("%s passed for %s", check.__class__.__name__, output_path) - - curr_exec_rec.complete_clean() - - # End. Return curr_run_step. Update the state. - if not bad_output_found and not bad_execution: - curr_run_step.finish_successfully(save=True) - else: - curr_run_step.finish_failure(save=True) - logger.debug("Quarantining any other RunComponents using the same ExecRecord") - curr_exec_rec.quarantine_runcomponents() # this is transaction'd - - curr_run_step.complete_clean() - return curr_run_step - - except KeyboardInterrupt: - # Execution was stopped somewhere outside of run_code (that would - # have been caught above). - curr_run_step.cancel_running(save=True) - raise StopExecution( - "Execution of step {} (method {}) was stopped during bookkeeping.".format( - curr_run_step, - curr_run_step.pipelinestep.transformation.definite - ) - ) - - -class RunPlan(object): - """ - Hold the plan for which steps will be executed in a sandbox. - - Also holds the dependencies between steps and cables, as well as the - ExecRecord that will be reused for each step and cable that doesn't have - to be run this time. - - This is required to avoid incoherencies where steps use different results from - the same preceding step in a Pipeline (for example, if that step is not deterministic. - """ - def load(self, top_level_run, inputs, subpipeline_step=None): - """ Load the steps from the pipeline and dataset dependencies. - - Links pipeline inputs and step outputs to the inputs of other steps. - - top_level_run refers to the top-level top_level_run. subpipeline is None or the PipelineStep - that represents the sub-Pipeline. This may be more than one layer deep in - the top-level Pipeline. - """ - self.top_level_run = top_level_run - self.run = top_level_run if not subpipeline_step else None - self.step_plans = [] - self.inputs = [DatasetPlan(input_item) for input_item in inputs] - if subpipeline_step: - assert subpipeline_step.transformation.is_pipeline() - self.pipeline = subpipeline_step.transformation.definite - else: - self.pipeline = top_level_run.pipeline - - for step in self.pipeline.steps.all(): - step_plan = StepPlan(step.step_num) - step_plan.pipeline_step = step - self.step_plans.append(step_plan) - for output in step.transformation.outputs.all(): - step_plan.outputs.append(DatasetPlan(step_num=step.step_num, - output_num=output.dataset_idx)) - for cable in step.cables_in.order_by("dest__dataset_idx"): - if cable.source_step == 0: - input_index = cable.source.definite.dataset_idx-1 - input_plan = self.inputs[input_index] - else: - step_index = cable.source_step-1 - output_index = cable.source.definite.dataset_idx-1 - input_plan = self.step_plans[step_index].outputs[output_index] - step_plan.inputs.append(input_plan) - - if step.is_subpipeline(): - step_plan.subrun_plan = RunPlan() - step_plan.subrun_plan.load(top_level_run, inputs, subpipeline_step=step) - - self.outputs = [] - for cable in self.pipeline.outcables.order_by("output_idx"): - step_index = cable.source_step-1 - output_index = cable.source.definite.dataset_idx-1 - output_plan = self.step_plans[step_index].outputs[output_index] - self.outputs.append(output_plan) - - def find_consistent_execution(self): - """ - Flesh out the plan for executing this run. - - First, StepPlans are created for each RunStep, creating RunSteps if necessary. - Then, suitable ExecRecords are identified where possible and added to the StepPlans. - Lastly, we iterate over all of the StepPlans until we've identified a consistent - plan for execution that will not lead to incoherent results. - """ - self.create_run_steps() - self.find_ERs() - self.identify_changes() - - def create_run_steps(self, subrun=None): - """ - Find or create the RunSteps in the Run. Sub-run RunPlans are also assigned Runs at this point. - """ - if subrun: - self.run = subrun - - for step_plan in self.step_plans: - step_subrun = None - run_step = self.run.runsteps.filter( - pipelinestep__step_num=step_plan.step_num).first() - if run_step is None: - run_step = RunStep.create(step_plan.pipeline_step, self.run, start=False) - - if run_step.pipelinestep.transformation.is_pipeline(): - step_subrun = Run( - user=self.top_level_run.user, - pipeline=run_step.pipelinestep.transformation.definite, - parent_runstep=run_step - ) - step_subrun.save() - step_subrun.users_allowed.add(*self.top_level_run.users_allowed.all()) - step_subrun.groups_allowed.add(*self.top_level_run.groups_allowed.all()) - - elif run_step.pipelinestep.transformation.is_pipeline(): - step_subrun = run_step.child_run - - if step_subrun: - step_plan.subrun_plan.create_run_steps(step_subrun) - - step_plan.run_step = run_step - - def find_ERs(self): - """ - Looks for suitable ExecRecords for the RunSteps and populates step_plan with them. - """ - for step_plan in self.step_plans: - if step_plan.pipeline_step.is_subpipeline(): - step_plan.subrun_plan.find_ERs() - continue - elif step_plan.run_step.execrecord is not None: - step_plan.execrecord = step_plan.run_step.execrecord - else: - input_datasets = [plan.dataset for plan in step_plan.inputs] - if all(input_datasets): - method = step_plan.pipeline_step.transformation.definite - if method.reusable == Method.NON_REUSABLE: - continue - execrecord, summary = step_plan.run_step.get_suitable_ER(input_datasets, reuse_failures=False) - - if not summary: - # no exec record, have to run - continue - if (not summary['fully reusable'] and - method.reusable != Method.DETERMINISTIC): - continue - step_plan.execrecord = execrecord - - if step_plan.execrecord is None: - # We couldn't find a suitable ExecRecord. - continue - - execrecordouts = step_plan.execrecord.execrecordouts.all() - for execrecordout in execrecordouts: - generic_output = execrecordout.generic_output - dataset_idx = generic_output.transformationoutput.dataset_idx - output = step_plan.outputs[dataset_idx-1] - output.dataset = execrecordout.dataset - - def identify_changes(self): - is_changed = True - while is_changed: - is_changed = self.walk_backward() or self.walk_forward() - - def walk_backward(self): - """ - Walk backward through the steps, flagging steps that need to be run. - - @return: True if any new steps were flagged for running. - """ - is_changed = False - for step_plan in reversed(self.step_plans): - if step_plan.pipeline_step.is_subpipeline(): - is_changed = step_plan.subrun_plan.walk_backward() or is_changed - - elif not step_plan.execrecord: - for input_plan in step_plan.inputs: - if not input_plan.has_data() and input_plan.step_num is not None: - # Note: if input_plan.step_num is None (i.e. this is an input to the Run) - # then we leave it -- it will be caught later in the execution. - - source_plan = self.step_plans[input_plan.step_num-1] - is_changed = source_plan.check_rerun(input_plan.output_num) or is_changed - - return is_changed - - def walk_forward(self): - """ - Walk forward through the steps, flagging steps that need to be run. - - @return: True if any new steps were flagged for running. - """ - is_changed = False - for step_plan in self.step_plans: - if step_plan.pipeline_step.is_subpipeline(): - is_changed = step_plan.subrun_plan.walk_forward() or is_changed - - else: - for input_plan in step_plan.inputs: - if not input_plan.dataset and step_plan.execrecord: - step_plan.execrecord = None - for output_plan in step_plan.outputs: - output_plan.dataset = None - is_changed = True - - return is_changed - - -class StepPlan(object): - """ Plan whether a step will actually be executed. - """ - def __init__(self, step_num): - self.step_num = step_num - self.execrecord = None - self.pipeline_step = None - self.run_step = None - self.inputs = [] - self.outputs = [] - - def check_rerun(self, output_idx=None): - """ - Check that this step can recreate one of its missing outputs. - - If the execrecord cannot be restored, don't use it, and mark all - the outputs as having no data. If this StepPlan is for a sub-Pipeline, - then output_idx must be specified; otherwise, it is ignored. - - @return: True if the execrecord had to be abandoned. - """ - if self.pipeline_step.is_subpipeline(): - assert output_idx is not None - # Look up the step that produced this output. - curr_output_plan = self.subrun_plan.outputs[output_idx-1] - source_step_plan = self.subrun_plan.step_plans[curr_output_plan.step_num-1] - - return source_step_plan.check_rerun(output_idx=output_idx) - - else: - if not self.execrecord: - # It didn't have an ExecRecord so there is no change to whether this needs to be rerun or not. - return False - - # At this point, we know self.execrecord exists. - method = self.run_step.transformation.definite - if method.reusable == Method.DETERMINISTIC: - # There will be no trouble reusing the execrecord. - return False - - # At this point we know the Method is not deterministic, so we'll have to re-run it. - self.execrecord = None - for output_plan in self.outputs: - output_plan.dataset = None - return True - - def __repr__(self): - return 'StepPlan({})'.format(self.step_num) - - def __eq__(self, other): - return isinstance(other, StepPlan) and other.step_num == self.step_num - - def __hash__(self): - return hash(self.step_num) - - -class DatasetPlan(object): - def __init__(self, dataset=None, step_num=None, output_num=None): - self.dataset = dataset - self.step_num = step_num - self.output_num = output_num - - def has_data(self): - return self.dataset and self.dataset.has_data() - - def __repr__(self): - if self.dataset is not None: - args = repr(self.dataset) - elif self.step_num is not None: - args = 'step_num={}, output_num={}'.format(self.step_num, - self.output_num) - else: - args = '' - return 'DatasetPlan({})'.format(args) - - def __eq__(self, other): - if not isinstance(other, DatasetPlan): - return False - if self.dataset is not None or other.dataset is not None: - return self.dataset == other.dataset - if self.step_num is not None: - return (self.step_num == other.step_num and - self.output_num == other.output_num) - return other is self - - def __hash__(self): - if self.dataset is not None: - return hash(self.dataset) - if self.step_num is not None: - return hash((self.step_num, self.output_num)) - return hash(self) - - -# A simple struct that holds the information required to perform a RunStep. -class RunStepExecuteInfo: - def __init__(self, runstep, user, cable_info_list, execrecord, step_run_dir, log_dir, output_paths, - recovering_record=None): - """ - Constructor. - - INPUTS - cable_info_list: an ordered list of RunCableExecuteInfo objects, with each - one corresponding to the input cable of the step with the same index. - """ - self.runstep = runstep - self.user = user - self.cable_info_list = cable_info_list - self.execrecord = execrecord - self.step_run_dir = step_run_dir - self.log_dir = log_dir - self.recovering_record = recovering_record - self.output_paths = output_paths - # FIXME in the future this number may vary across runs. - self.threads_required = None - if not runstep.pipelinestep.is_subpipeline(): - self.threads_required = runstep.transformation.definite.threads - - def flag_for_recovery(self, recovering_record): - assert self.recovering_record is None - self.recovering_record = recovering_record - - def is_recovery(self): - return self.recovering_record is not None - - def dict_repr(self): - return { - "runstep_pk": self.runstep.pk, - "user_pk": self.user.pk, - "cable_info_dicts": [x.dict_repr() for x in self.cable_info_list], - "execrecord_pk": None if self.execrecord is None else self.execrecord.pk, - "step_run_dir": self.step_run_dir, - "log_dir": self.log_dir, - "recovering_record_pk": self.recovering_record.pk if self.recovering_record is not None else None, - "output_paths": self.output_paths, - "threads_required": self.threads_required - } - - def driver_stdout_path_prefix(self): - """ - Return the filename prefix for the stdout log file. - - This is used not only to assemble the actual path of the file, but also - for finding it in the sandbox after it's complete, as the actual path - contains some Slurm macros. - """ - return "step{}_stdout".format(self.runstep.pipelinestep.step_num) - - def driver_stderr_path_prefix(self): - """ - Counterpart to driver_stdout_path_prefix for the stderr log file. - """ - return "step{}_stderr".format(self.runstep.pipelinestep.step_num) - - def driver_stdout_path(self): - return os.path.join( - self.log_dir, - "{}_slurmID%J_node%N.txt".format(self.driver_stdout_path_prefix()) - ) - - def driver_stderr_path(self): - return os.path.join( - self.log_dir, - "{}_slurmID%J_node%N.txt".format(self.driver_stderr_path_prefix()) - ) - - def setup_stdout_path(self): - return os.path.join(self.log_dir, "setup_out_slurmID%J_node%N.txt") - - def setup_stderr_path(self): - return os.path.join(self.log_dir, "setup_err_slurmID%J_node%N.txt") - - def bookkeeping_stdout_path(self): - return os.path.join(self.log_dir, "bookkeeping_out_slurmID%J_node%N.txt") - - def bookkeeping_stderr_path(self): - return os.path.join(self.log_dir, "bookkeeping_err_slurmID%J_node%N.txt") - - -class RunCableExecuteInfo: - def __init__(self, cable_record, user, execrecord, input_dataset, input_dataset_path, output_path, - log_dir, recovering_record=None, by_step=None, could_be_reused=False): - """ - Constructor. - """ - self.cable_record = cable_record - self.user = user - self.execrecord = execrecord - self.input_dataset = input_dataset - self.input_dataset_path = input_dataset_path - self.output_path = output_path - self.recovering_record = recovering_record - self.by_step = by_step - # FIXME will we ever need more than 1 thread for this? - self.threads_required = 1 - self.ready_to_go = False - self.cancelled = False - self.could_be_reused = could_be_reused - self.cable_info_dir = None - self.log_dir = log_dir - - def flag_for_recovery(self, recovering_record, by_step=None): - assert self.recovering_record is None - self.recovering_record = recovering_record - self.by_step = by_step - - def cancel(self): - self.cancelled = True - - def is_recovery(self): - return self.recovering_record is not None - - def dict_repr(self): - return { - "cable_record_pk": self.cable_record.pk, - "user_pk": self.user.pk, - "execrecord_pk": self.execrecord.pk if self.execrecord is not None else None, - "input_dataset_pk": self.input_dataset.pk, - "input_dataset_path": self.input_dataset_path, - "output_path": self.output_path, - "recovering_record_pk": self.recovering_record.pk if self.recovering_record is not None else None, - "by_step_pk": None if self.by_step is None else self.by_step.pk, - "threads_required": self.threads_required, - "ready_to_go": self.ready_to_go, - "cable_info_dir": self.cable_info_dir, - "log_dir": self.log_dir - } - - def stdout_prefix(self): - """ - Reports the prefix of the stdout log file that should be produced. - - This is useful because we use this to identify the resulting log file, - which will have some Slurm-specific information added to it. - """ - if isinstance(self.cable_record, RunSIC): - cable_idx = self.cable_record.component.dest.dataset_idx - cable_type_str = "input" - else: - cable_idx = self.cable_record.component.source.dataset_idx - cable_type_str = "output" - - return "{}{}_stdout".format(cable_type_str, cable_idx) - - def stderr_prefix(self): - """ - Counterpart of stdout_prefix for the stderr log file. - """ - if isinstance(self.cable_record, RunSIC): - cable_idx = self.cable_record.component.dest.dataset_idx - cable_type_str = "input" - else: - cable_idx = self.cable_record.component.source.dataset_idx - cable_type_str = "output" - - return "{}{}_stderr".format(cable_type_str, cable_idx) - - def stdout_path(self): - """ - Produces the actual path with Slurm macros that will be used for stdout logging. - """ - return os.path.join(self.log_dir, "{}_slurmID%J_node%N.txt".format(self.stdout_prefix())) - - def stderr_path(self): - """ - Produces the actual path with Slurm macros that will be used for stderr logging. - """ - return os.path.join(self.log_dir, "{}_slurmID%J_node%N.txt".format(self.stderr_prefix())) - - def set_cable_info_dir(self, cable_info_dir): - self.cable_info_dir = cable_info_dir diff --git a/kive/sandbox/tests.py b/kive/sandbox/tests.py deleted file mode 100644 index 914737441..000000000 --- a/kive/sandbox/tests.py +++ /dev/null @@ -1,1852 +0,0 @@ -import os -import re -import sys -import tempfile -import shutil - -from mock import call, patch - -from django.contrib.auth.models import User -from django.core.files import File -from django.core.files.base import ContentFile -from django.test import TestCase, skipIfDBFeature -from django.utils import timezone -from django.conf import settings - -from archive.models import Run, RunComponent -from constants import datatypes -from datachecking.models import IntegrityCheckLog, MD5Conflict -from kive.testing_utils import clean_up_all_files -from kive.tests import install_fixture_files, remove_fixture_files, BaseTestCases -from librarian.models import Dataset, DatasetStructure, ExternalFileDirectory, ExecRecord -from metadata.models import Datatype, CompoundDatatype, everyone_group -from method.models import CodeResource, CodeResourceRevision, Method, MethodFamily -from method.tests import samplecode_path -from pipeline.models import Pipeline, PipelineFamily -from sandbox.execute import Sandbox, RunPlan, StepPlan, DatasetPlan -from fleet.workers import Manager -import file_access_utils - - -def execute_tests_environment_setup(case): - print("FOOOOOOBAR {}".format(settings.MEDIA_ROOT)) - # Users + method/pipeline families - case.myUser = User.objects.create_user('john', 'lennon@thebeatles.com', 'johnpassword') - case.myUser.save() - case.myUser.groups.add(everyone_group()) - case.myUser.save() - - case.mf = MethodFamily(name="self.mf", description="self.mf desc", user=case.myUser) - case.mf.save() - case.pf = PipelineFamily(name="self.pf", description="self.pf desc", user=case.myUser) - case.pf.save() - - # Code on file system - case.mA_cr = CodeResource(name="mA_CR", description="self.mA_cr desc", filename="mA.py", user=case.myUser) - case.mA_cr.save() - case.mA_crr = CodeResourceRevision(coderesource=case.mA_cr, revision_name="v1", revision_desc="desc", - user=case.myUser) - with open(os.path.join(samplecode_path, "generic_script.py"), "rb") as f: - new_file_MD5 = file_access_utils.compute_md5(f) - f.seek(0) - case.mA_crr.content_file.save("generic_script.py", File(f)) - case.mA_crr.MD5_checksum = new_file_MD5 - - case.mA_crr.save() - - # Basic DTs - case.string_dt = Datatype.objects.get(pk=datatypes.STR_PK) - case.int_dt = Datatype.objects.get(pk=datatypes.INT_PK) - - # Basic CDTs - case.pX_in_cdt = CompoundDatatype(user=case.myUser) - case.pX_in_cdt.save() - case.pX_in_cdtm_1 = case.pX_in_cdt.members.create(datatype=case.int_dt, column_name="pX_a", column_idx=1) - case.pX_in_cdtm_2 = case.pX_in_cdt.members.create(datatype=case.int_dt, column_name="pX_b", column_idx=2) - case.pX_in_cdtm_3 = case.pX_in_cdt.members.create(datatype=case.string_dt, column_name="pX_c", column_idx=3) - - case.mA_in_cdt = CompoundDatatype(user=case.myUser) - case.mA_in_cdt.save() - case.mA_in_cdtm_1 = case.mA_in_cdt.members.create(datatype=case.string_dt, column_name="a", column_idx=1) - case.mA_in_cdtm_2 = case.mA_in_cdt.members.create(datatype=case.int_dt, column_name="b", column_idx=2) - - case.mA_out_cdt = CompoundDatatype(user=case.myUser) - case.mA_out_cdt.save() - case.mA_out_cdtm_1 = case.mA_out_cdt.members.create(datatype=case.int_dt, column_name="c", column_idx=1) - case.mA_out_cdtm_2 = case.mA_out_cdt.members.create(datatype=case.string_dt, column_name="d", column_idx=2) - - case.dataset = Dataset.create_dataset( - os.path.join(samplecode_path, "input_for_test_C_twostep_with_subpipeline.csv"), - user=case.myUser, - cdt=case.pX_in_cdt, - name="pX_in_dataset", - description="input to pipeline pX" - ) - case.raw_dataset = Dataset.create_dataset( - os.path.join(samplecode_path, "input_for_test_C_twostep_with_subpipeline.csv"), - user=case.myUser, - cdt=None, - name="pX_in_dataset", - description="input to pipeline pX" - ) - - # Method + input/outputs - case.mA = Method(revision_name="mA", revision_desc="mA_desc", family=case.mf, driver=case.mA_crr, - user=case.myUser) - case.mA.save() - case.mA_in = case.mA.create_input(compounddatatype=case.mA_in_cdt, dataset_name="mA_in", dataset_idx=1) - case.mA_out = case.mA.create_output(compounddatatype=case.mA_out_cdt, dataset_name="mA_out", dataset_idx=1) - - # Define pipeline containing the method, and its input + outcables - case.pX = Pipeline(family=case.pf, revision_name="pX_revision", revision_desc="X", - user=case.myUser) - case.pX.save() - case.X1_in = case.pX.create_input(compounddatatype=case.pX_in_cdt, dataset_name="pX_in", dataset_idx=1) - case.step_X1 = case.pX.steps.create(transformation=case.mA, step_num=1) - - # Custom cable from pipeline input to method - case.cable_X1_A1 = case.step_X1.cables_in.create(dest=case.mA_in, source_step=0, source=case.X1_in) - case.wire1 = case.cable_X1_A1.custom_wires.create(source_pin=case.pX_in_cdtm_2, dest_pin=case.mA_in_cdtm_2) - case.wire2 = case.cable_X1_A1.custom_wires.create(source_pin=case.pX_in_cdtm_3, dest_pin=case.mA_in_cdtm_1) - - # Pipeline outcables - case.X1_outcable = case.pX.create_outcable(output_name="pX_out", output_idx=1, source_step=1, - source=case.mA_out) - case.pX.create_outputs() - - # Pipeline with raw input. - pX_raw = Pipeline(family=case.pf, revision_name="pX_raw", revision_desc="X", user=case.myUser) - pX_raw.save() - mA_raw = Method(revision_name="mA_raw", revision_desc="mA_desc", family=case.mf, driver=case.mA_crr, - user=case.myUser) - mA_raw.save() - mA_in_raw = mA_raw.create_input(compounddatatype=None, dataset_name="mA_in", dataset_idx=1) - mA_out_raw = mA_raw.create_output(compounddatatype=case.mA_out_cdt, dataset_name="mA_out", dataset_idx=1) - X1_in_raw = pX_raw.create_input(compounddatatype=None, dataset_name="pX_in", dataset_idx=1) - step_X1_raw = pX_raw.steps.create(transformation=mA_raw, step_num=1) - step_X1_raw.cables_in.create(dest=mA_in_raw, source_step=0, source=X1_in_raw) - pX_raw.create_outcable(output_name="pX_out", output_idx=1, source_step=1, source=mA_out_raw) - pX_raw.create_outputs() - - -def execute_tests_environment_load(case): - case.myUser = User.objects.get(is_staff=False) - case.mf = MethodFamily.objects.get() - case.pf = PipelineFamily.objects.get() - case.mA_cr = CodeResource.objects.get() - case.mA_crr = CodeResourceRevision.objects.get() - case.string_dt = Datatype.objects.get(pk=datatypes.STR_PK) - case.int_dt = Datatype.objects.get(pk=datatypes.INT_PK) - case.pX_in_cdt = CompoundDatatype.objects.get(members__column_name='pX_a') - case.pX_in_cdtm_1 = case.pX_in_cdt.members.get(column_name="pX_a") - case.pX_in_cdtm_2 = case.pX_in_cdt.members.get(column_name="pX_b") - case.pX_in_cdtm_3 = case.pX_in_cdt.members.get(column_name="pX_c") - - case.mA_in_cdt = CompoundDatatype.objects.get(members__column_name='a') - case.mA_in_cdtm_1 = case.mA_in_cdt.members.get(column_name="a") - case.mA_in_cdtm_2 = case.mA_in_cdt.members.get(column_name="b") - - case.mA_out_cdt = CompoundDatatype.objects.get(members__column_name='c') - case.mA_out_cdtm_1 = case.mA_out_cdt.members.get(column_name="c") - case.mA_out_cdtm_2 = case.mA_out_cdt.members.get(column_name="d") - - case.dataset = Dataset.objects.get( - structure__compounddatatype=case.pX_in_cdt) - case.raw_dataset = Dataset.objects.get(structure__isnull=True) - case.mA = Method.objects.get(revision_name="mA") - case.mA_in = case.mA.inputs.get() - case.mA_out = case.mA.outputs.get() - - case.pX = Pipeline.objects.get(revision_name="pX_revision") - case.X1_in = case.pX.inputs.get() - case.step_X1 = case.pX.steps.get() - - case.cable_X1_A1 = case.step_X1.cables_in.get() - case.wire1 = case.cable_X1_A1.custom_wires.get(source_pin=case.pX_in_cdtm_2) - case.wire2 = case.cable_X1_A1.custom_wires.get(source_pin=case.pX_in_cdtm_3) - - case.X1_outcable = case.pX.outcables.get() - case.pX_raw = Pipeline.objects.get(revision_name="pX_raw") - - -@skipIfDBFeature('is_mocked') -class ExecuteTestsBase(BaseTestCases.SlurmExecutionTestCase): - fixtures = ['execute_tests'] - - def setUp(self): - install_fixture_files("execute_tests") - execute_tests_environment_load(self) - - def tearDown(self): - clean_up_all_files() - remove_fixture_files() - - def find_raw_pipeline(self, user): - """Find a Pipeline with a raw input.""" - for p in Pipeline.filter_by_user(user): - for input in p.inputs.all(): - if input.is_raw(): - return p - - def find_nonraw_pipeline(self, user): - """Find a Pipeline with no raw input.""" - for p in Pipeline.filter_by_user(user): - none_raw = True - for input in p.inputs.all(): - if input.is_raw(): - none_raw = False - break - if none_raw: - return p - - def find_inputs_for_pipeline(self, pipeline): - """Find appropriate input Datasets for a Pipeline.""" - input_datasets = [] - pipeline_owner = pipeline.user - for input in pipeline.inputs.all(): - if input.is_raw(): - candidate_datasets = Dataset.objects.filter(user=pipeline_owner) - if candidate_datasets.exists(): - for dataset in candidate_datasets: - if dataset.is_raw(): - dataset = dataset - break - else: - dataset = None - else: - datatype = input.structure.compounddatatype - structure = DatasetStructure.objects.filter( - compounddatatype=datatype, dataset__user=pipeline_owner) - if structure.exists(): - dataset = structure.first().dataset - else: - dataset = None - input_datasets.append(dataset) - return input_datasets - - -class ExecuteTests(ExecuteTestsBase): - - def test_pipeline_execute_A_simple_onestep_pipeline(self): - """Execution of a one-step pipeline.""" - - # Execute pipeline - pipeline = self.pX - inputs = [self.dataset] - run = Manager.execute_pipeline(self.myUser, pipeline, inputs).get_last_run() - - self.check_run_OK(run) - - def test_pipeline_execute_B_twostep_pipeline_with_recycling(self): - """Two step pipeline with second step identical to the first""" - - # Define pipeline containing two steps with the same method + pipeline input - self.pX = Pipeline(family=self.pf, revision_name="pX_revision", revision_desc="X", user=self.myUser) - self.pX.save() - self.X1_in = self.pX.create_input(compounddatatype=self.pX_in_cdt, dataset_name="pX_in", dataset_idx=1) - self.step_X1 = self.pX.steps.create(transformation=self.mA, step_num=1) - self.step_X2 = self.pX.steps.create(transformation=self.mA, step_num=2) - - # Use the SAME custom cable from pipeline input to steps 1 and 2 - self.cable_X1_A1 = self.step_X1.cables_in.create(dest=self.mA_in, source_step=0, source=self.X1_in) - self.wire1 = self.cable_X1_A1.custom_wires.create(source_pin=self.pX_in_cdtm_2, dest_pin=self.mA_in_cdtm_2) - self.wire2 = self.cable_X1_A1.custom_wires.create(source_pin=self.pX_in_cdtm_3, dest_pin=self.mA_in_cdtm_1) - self.cable_X1_A2 = self.step_X2.cables_in.create(dest=self.mA_in, source_step=0, source=self.X1_in) - self.wire3 = self.cable_X1_A2.custom_wires.create(source_pin=self.pX_in_cdtm_2, dest_pin=self.mA_in_cdtm_2) - self.wire4 = self.cable_X1_A2.custom_wires.create(source_pin=self.pX_in_cdtm_3, dest_pin=self.mA_in_cdtm_1) - - # POCs: one is trivial, the second uses custom outwires - # Note: by default, create_outcables assumes the POC has the CDT of the source (IE, this is a TRIVIAL cable) - self.outcable_1 = self.pX.create_outcable(output_name="pX_out_1", - output_idx=1, - source_step=1, - source=self.mA_out) - self.outcable_2 = self.pX.create_outcable(output_name="pX_out_2", - output_idx=2, - source_step=2, - source=self.mA_out) - - # Define CDT for the second output (first output is defined by a trivial cable) - self.pipeline_out2_cdt = CompoundDatatype(user=self.myUser) - self.pipeline_out2_cdt.save() - self.out2_cdtm_1 = self.pipeline_out2_cdt.members.create(column_name="c", column_idx=1, datatype=self.int_dt) - self.out2_cdtm_2 = self.pipeline_out2_cdt.members.create(column_name="d", column_idx=2, datatype=self.string_dt) - self.out2_cdtm_3 = self.pipeline_out2_cdt.members.create(column_name="e", column_idx=3, datatype=self.string_dt) - - # Second cable is not a trivial - we assign the new CDT to it - self.outcable_2.output_cdt = self.pipeline_out2_cdt - self.outcable_2.save() - - # Define custom outwires to the second output (Wire twice from cdtm 2) - self.outwire1 = self.outcable_2.custom_wires.create(source_pin=self.mA_out_cdtm_1, dest_pin=self.out2_cdtm_1) - self.outwire2 = self.outcable_2.custom_wires.create(source_pin=self.mA_out_cdtm_2, dest_pin=self.out2_cdtm_2) - self.outwire3 = self.outcable_2.custom_wires.create(source_pin=self.mA_out_cdtm_2, dest_pin=self.out2_cdtm_3) - - # Have the cables define the TOs of the pipeline - self.pX.create_outputs() - - # Execute pipeline - pipeline = self.pX - inputs = [self.dataset] - run = Manager.execute_pipeline(self.myUser, pipeline, inputs).get_last_run() - - self.check_run_OK(run) - - def test_pipeline_execute_C_twostep_pipeline_with_subpipeline(self): - """Two step pipeline with second step identical to the first""" - - # Define 2 member input and 1 member output CDTs for inner pipeline pY - self.pY_in_cdt = CompoundDatatype(user=self.myUser) - self.pY_in_cdt.save() - self.pY_in_cdtm_1 = self.pY_in_cdt.members.create(column_name="pYA", column_idx=1, datatype=self.int_dt) - self.pY_in_cdtm_2 = self.pY_in_cdt.members.create(column_name="pYB", column_idx=2, datatype=self.string_dt) - - self.pY_out_cdt = CompoundDatatype(user=self.myUser) - self.pY_out_cdt.save() - self.pY_out_cdt_cdtm_1 = self.pY_out_cdt.members.create(column_name="pYC", column_idx=1, datatype=self.int_dt) - - # Define 1-step inner pipeline pY - self.pY = Pipeline(family=self.pf, revision_name="pY_revision", revision_desc="Y", user=self.myUser) - self.pY.save() - self.pY_in = self.pY.create_input(compounddatatype=self.pY_in_cdt, dataset_name="pY_in", dataset_idx=1) - - self.pY_step_1 = self.pY.steps.create(transformation=self.mA, step_num=1) - self.pY_cable_in = self.pY_step_1.cables_in.create(dest=self.mA_in, source_step=0, source=self.pY_in) - self.pY_cable_in.custom_wires.create(source_pin=self.pY_in_cdtm_1, dest_pin=self.mA_in_cdtm_2) - self.pY_cable_in.custom_wires.create(source_pin=self.pY_in_cdtm_2, dest_pin=self.mA_in_cdtm_1) - - self.pY_cable_out = self.pY.outcables.create( - output_name="pY_out", output_idx=1, source_step=1, - source=self.mA_out, output_cdt=self.pY_out_cdt - ) - self.pY_outwire1 = self.pY_cable_out.custom_wires.create(source_pin=self.mA_out_cdtm_1, - dest_pin=self.pY_out_cdt_cdtm_1) - self.pY.create_outputs() - - # Define CDTs for the output of pX - self.pX_out_cdt_1 = CompoundDatatype(user=self.myUser) - self.pX_out_cdt_1.save() - self.pX_out_cdt_1_cdtm_1 = self.pX_out_cdt_1.members.create(column_name="pXq", column_idx=1, - datatype=self.int_dt) - - self.pX_out_cdt_2 = CompoundDatatype(user=self.myUser) - self.pX_out_cdt_2.save() - self.pX_out_cdt_2_cdtm_1 = self.pX_out_cdt_2.members.create( - column_name="pXr", column_idx=1, datatype=self.string_dt - ) - - # Define outer 2-step pipeline with mA at step 1 and pY at step 2 - self.pX = Pipeline(family=self.pf, revision_name="pX_revision", revision_desc="X", user=self.myUser) - self.pX.save() - self.X1_in = self.pX.create_input(compounddatatype=self.pX_in_cdt, dataset_name="pX_in", dataset_idx=1) - self.pX_step_1 = self.pX.steps.create(transformation=self.mA, step_num=1) - self.pX_step_2 = self.pX.steps.create(transformation=self.pY, step_num=2) - - self.pX_step_1_cable = self.pX_step_1.cables_in.create(dest=self.mA_in, source_step=0, source=self.X1_in) - self.pX_step_1_cable.custom_wires.create(source_pin=self.pX_in_cdtm_2, dest_pin=self.mA_in_cdtm_2) - self.pX_step_1_cable.custom_wires.create(source_pin=self.pX_in_cdtm_3, dest_pin=self.mA_in_cdtm_1) - - self.pX_step_2_cable = self.pX_step_2.cables_in.create(dest=self.pY_in, source_step=1, source=self.mA_out) - self.pX_step_2_cable.custom_wires.create(source_pin=self.mA_out_cdtm_1, dest_pin=self.pY_in_cdtm_1) - self.pX_step_2_cable.custom_wires.create(source_pin=self.mA_out_cdtm_2, dest_pin=self.pY_in_cdtm_2) - - self.pX_outcable_1 = self.pX.outcables.create( - output_name="pX_out_1", output_idx=1, source_step=1, - source=self.mA_out, output_cdt=self.pX_out_cdt_2 - ) - self.pX_outcable_1.custom_wires.create(source_pin=self.mA_out_cdtm_2, dest_pin=self.pX_out_cdt_2_cdtm_1) - - self.pX_outcable_2 = self.pX.outcables.create( - output_name="pX_out_2", output_idx=2, source_step=2, - source=self.pY.outputs.get(dataset_name="pY_out"), output_cdt=self.pX_out_cdt_1) - self.pX_outcable_2.custom_wires.create( - source_pin=self.pY.outputs.get(dataset_name="pY_out").get_cdt().members.get(column_name="pYC"), - dest_pin=self.pX_out_cdt_1_cdtm_1 - ) - - self.pX.create_outputs() - - # Dataset for input during execution of pipeline - input_dataset = Dataset.create_dataset( - os.path.join(samplecode_path, "input_for_test_C_twostep_with_subpipeline.csv"), - user=self.myUser, - cdt=self.pX_in_cdt, - keep_file=True, - name="input_dataset", - description="dataset description" - ) - - # Execute pipeline - pipeline = self.pX - inputs = [input_dataset] - run = Manager.execute_pipeline(self.myUser, pipeline, inputs).get_last_run() - self.check_run_OK(run) - - def test_pipeline_all_inputs_OK_nonraw(self): - """Execute a Pipeline with OK non-raw inputs.""" - pipeline = self.find_nonraw_pipeline(self.myUser) - inputs = self.find_inputs_for_pipeline(pipeline) - self.assertTrue(all(i.is_OK() for i in inputs)) - self.assertFalse(all(i.is_raw() for i in inputs)) - run = Manager.execute_pipeline(self.myUser, pipeline, inputs).get_last_run() - - self.check_run_OK(run) - - # Check the full is_complete and is_successful. - self.assertTrue(run.is_complete()) - self.assertTrue(run.is_successful()) - - self.assertIsNone(run.clean()) - self.assertIsNone(run.complete_clean()) - - def test_pipeline_all_inputs_OK_raw(self): - """Execute a Pipeline with OK raw inputs.""" - # Find a Pipeline with a raw input. - pipeline = self.find_raw_pipeline(self.myUser) - self.assertIsNotNone(pipeline) - inputs = self.find_inputs_for_pipeline(pipeline) - self.assertTrue(all(i.is_OK() for i in inputs)) - self.assertTrue(any(i.is_raw() for i in inputs)) - run = Manager.execute_pipeline(self.myUser, pipeline, inputs).get_last_run() - - self.check_run_OK(run) - - self.assertTrue(run.is_complete()) - self.assertTrue(run.is_successful()) - - self.assertIsNone(run.clean()) - self.assertIsNone(run.complete_clean()) - - def test_pipeline_all_inputs_initially_OK(self): - """Execute a Pipeline with inputs that were initially OK but have a failed integrity check.""" - pipeline = self.find_nonraw_pipeline(self.myUser) - inputs = self.find_inputs_for_pipeline(pipeline) - self.assertTrue(all(i.is_OK() for i in inputs)) - self.assertFalse(all(i.is_raw() for i in inputs)) - - # Spoil one of the inputs with a bad integrity check. - now = timezone.now() - for i, dataset in enumerate(inputs, start=1): - bad_input = dataset - bad_icl = IntegrityCheckLog(dataset=dataset, - user=self.myUser, - start_time=now, - end_time=now) - bad_icl.save() - MD5Conflict(integritychecklog=bad_icl, conflicting_dataset=dataset).save() - break - self.assertFalse(bad_input.is_OK()) - - run = Manager.execute_pipeline(self.myUser, pipeline, inputs).get_last_run() - - self.check_run_OK(run) - - # Check the full is_complete and is_successful. - self.assertTrue(run.is_complete()) - self.assertTrue(run.is_successful()) - - self.assertIsNone(run.clean()) - self.assertIsNone(run.complete_clean()) - - # The spoiled input should have been re-validated. - bad_input.refresh_from_db() - self.assertTrue(bad_input.is_OK()) - - def test_pipeline_inputs_not_initially_OK(self): - """Can't execute a Pipeline with non-OK non-raw inputs.""" - pipeline = self.find_nonraw_pipeline(self.myUser) - inputs = self.find_inputs_for_pipeline(pipeline) - self.assertTrue(all(i.is_OK() for i in inputs)) - self.assertFalse(all(i.is_raw() for i in inputs)) - - for i, dataset in enumerate(inputs, start=1): - if not dataset.is_raw() and dataset.content_checks.count() == 1: - bad_input, bad_index = dataset, i - orig_ccl = dataset.content_checks.first() - orig_ccl.add_missing_output() - break - self.assertFalse(bad_input.initially_OK()) - self.assertFalse(bad_input.is_OK()) - - self.assertRaisesRegexp( - ValueError, - re.escape('Dataset {} passed as input {} to Pipeline "{}" was not initially OK' - .format(bad_input, bad_index, pipeline)), - lambda: Manager.execute_pipeline(self.myUser, pipeline, inputs) - ) - - def test_crr_corrupted(self): - """ - Test that a Run fails if a CodeResource has been corrupted. - """ - self.mA_crr.content_file.save("NowCorrupted.dat", ContentFile("CORRUPTED")) - self.mA_crr.save() - - run = Manager.execute_pipeline(self.myUser, self.pX, [self.dataset]).get_last_run() - - # This Run should have failed right away. - rs = run.runsteps.first() - - self.assertFalse(rs.log.methodoutput.are_checksums_OK) - - self.assertTrue(rs.is_failed()) - - self.assertTrue(run.is_failed()) - - for cancelled_rs in run.runsteps.exclude(pk=rs.pk): - self.assertTrue(cancelled_rs.is_cancelled()) - - # FIXME this test revealed issues #534 and #535; when we fix these, revisit this test. - # def test_filling_in_execrecord_with_incomplete_content_check(self): - # """Execution that fills in an ExecRecord that doesn't have a complete content check.""" - # - # # Execute pipeline - # pipeline = self.pX - # inputs = [self.dataset] - # run = Manager.execute_pipeline(self.myUser, pipeline, inputs).get_last_run() - # - # # This was one step, so we go into that first step and fiddle with the ContentCheckLog. - # rs = run.runsteps.first() - # ccl_to_alter = rs.execrecord.execrecordouts.first().dataset.content_checks.first() - # ccl_to_alter.end_time = None - # ccl_to_alter.start_time = None - # ccl_to_alter.save() - # - # # Now execute the pipeline again. - # run2 = Manager.execute_pipeline(self.myUser, pipeline, inputs).get_last_run() - # r2s = run2.runsteps.first() - # # It should have filled in the same execrecord. - # self.assertEquals(r2s.execrecord, rs.execrecord) - # # There should be an integrity check and a content check both associated to r2s' log. - # self.assertEquals(r2s.log.integrity_check.count(), 1) - # self.assertEquals(r2s.log.content_check.count(), 1) - - def test_filling_in_execrecord_with_incomplete_content_check(self): - """Execution that fills in an ExecRecord that doesn't have a complete content check.""" - - # Execute pipeline - pipeline = self.pX_raw - inputs = [self.raw_dataset] - run = Manager.execute_pipeline(self.myUser, pipeline, inputs).get_last_run() - - # This was one step, so we go into that first step and fiddle with the ContentCheckLog. - rs = run.runsteps.first() - ccl_to_alter = rs.execrecord.execrecordouts.first().dataset.content_checks.first() - ccl_to_alter.end_time = None - ccl_to_alter.start_time = None - ccl_to_alter.save() - - # Now we dummy it up to look like the RunStep never finished, so no RunOutputCable was run - # and rs is not marked complete. - roc = run.runoutputcables.first() - roc.delete() - rs._complete = False - rs.save() - - # Now execute the pipeline again. - run2 = Manager.execute_pipeline(self.myUser, pipeline, inputs).get_last_run() - r2s = run2.runsteps.first() - # It should have filled in the same execrecord. - self.assertEquals(r2s.execrecord, rs.execrecord) - # There should be an integrity check and a content check both associated to r2s' log. - self.assertEquals(r2s.log.integrity_checks.count(), 1) - self.assertEquals(r2s.log.content_checks.count(), 1) - - @patch.object(Dataset, "attempt_to_decontaminate_runcomponents_using_as_output", autospec=True, - side_effect=Dataset.attempt_to_decontaminate_runcomponents_using_as_output) - @patch.object(Dataset, "quarantine_runcomponents_using_as_output", autospec=True, - side_effect=Dataset.quarantine_runcomponents_using_as_output) - @patch.object(ExecRecord, "attempt_decontamination", autospec=True, - side_effect=ExecRecord.attempt_decontamination) - @patch.object(ExecRecord, "quarantine_runcomponents", autospec=True, - side_effect=ExecRecord.quarantine_runcomponents) - @patch.object(Run, "attempt_decontamination", autospec=True, side_effect=Run.attempt_decontamination) - @patch.object(RunComponent, "decontaminate", autospec=True, side_effect=RunComponent.decontaminate) - @patch.object(Run, "quarantine", autospec=True, side_effect=Run.quarantine) - @patch.object(Run, "mark_failure", autospec=True, side_effect=Run.mark_failure) - @patch.object(Run, "stop", autospec=True, side_effect=Run.stop) - @patch.object(Run, "start", autospec=True, side_effect=Run.start) - @patch.object(RunComponent, "cancel_running", autospec=True, side_effect=RunComponent.cancel_running) - @patch.object(RunComponent, "cancel_pending", autospec=True, side_effect=RunComponent.cancel_pending) - @patch.object(RunComponent, "begin_recovery", autospec=True, side_effect=RunComponent.begin_recovery) - @patch.object(RunComponent, "quarantine", autospec=True, side_effect=RunComponent.quarantine) - @patch.object(RunComponent, "finish_failure", autospec=True, side_effect=RunComponent.finish_failure) - @patch.object(RunComponent, "finish_successfully", autospec=True, side_effect=RunComponent.finish_successfully) - @patch.object(RunComponent, "start", autospec=True, side_effect=RunComponent.start) - def test_execution_decontaminates_quarantined_runsteps( - self, - mock_start, - mock_finish_successfully, - mock_finish_failure, - mock_quarantine, - mock_begin_recovery, - mock_cancel_pending, - mock_cancel_running, - mock_run_start, - mock_run_stop, - mock_run_mark_failure, - mock_run_quarantine, - mock_decontaminate, - mock_run_attempt_decontamination, - mock_execrecord_quarantine_runcomponents, - mock_execrecord_attempt_decontamination, - mock_dataset_quarantine_rcs, - mock_dataset_attempt_decontamination - ): - """ - Executing a pipeline with a quarantined component properly re-validates it. - """ - # Start by executing a simple one-step pipeline. - pipeline = self.pX_raw - inputs = [self.raw_dataset] - run1 = Manager.execute_pipeline(self.myUser, pipeline, inputs).get_last_run() - self.check_run_OK(run1) - - run1_step1 = run1.runsteps.get(pipelinestep__step_num=1) - run1_outcable = run1.runoutputcables.first() - - # All of the RunComponents should have been started. - mock_start.assert_has_calls([ - call(run1_step1), - call(run1_step1.RSICs.first()), - call(run1_outcable) - ]) - self.assertEquals(mock_start.call_count, 3) - - # All of them should have been finished successfully without event. - mock_finish_successfully.assert_has_calls([ - call(run1_step1.RSICs.first(), save=True), - call(run1_step1, save=True), - call(run1_outcable, save=True) - ]) - self.assertEquals(mock_finish_successfully.call_count, 3) - - # These were not called, so have not been mocked yet. - self.assertFalse(hasattr(mock_finish_failure, "assert_not_called")) - self.assertFalse(hasattr(mock_quarantine, "assert_not_called")) - self.assertFalse(hasattr(mock_begin_recovery, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_pending, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_running, "assert_not_called")) - self.assertFalse(hasattr(mock_run_quarantine, "assert_not_called")) - self.assertFalse(hasattr(mock_decontaminate, "assert_not_called")) - self.assertFalse(hasattr(mock_run_attempt_decontamination, "assert_not_called")) - self.assertFalse(hasattr(mock_execrecord_quarantine_runcomponents, "assert_not_called")) - self.assertFalse(hasattr(mock_execrecord_attempt_decontamination, "assert_not_called")) - self.assertFalse(hasattr(mock_dataset_quarantine_rcs, "assert_not_called")) - self.assertFalse(hasattr(mock_dataset_attempt_decontamination, "assert_not_called")) - - mock_run_start.assert_called_once_with(run1, save=True) - mock_run_stop.assert_called_once_with(run1, save=True) - self.assertFalse(hasattr(mock_run_mark_failure, "assert_not_called")) - - mock_run_start.reset_mock() - mock_run_stop.reset_mock() - mock_start.reset_mock() - mock_finish_successfully.reset_mock() - - # Now, let's invalidate the output of the first step. - run1_rs = run1.runsteps.first() - run1_outcable = run1.runoutputcables.first() - step1_orig_output = run1_rs.outputs.first() - - corrupted_contents = "Corrupted file" - _, temp_file_path = tempfile.mkstemp() - with open(temp_file_path, "wb") as f: - f.write(corrupted_contents) - step1_orig_output.check_integrity(temp_file_path, self.myUser, notify_all=True) - os.remove(temp_file_path) - - # This should have quarantined run1_rs and run1. - run1.refresh_from_db() - run1_rs.refresh_from_db() - run1_outcable.refresh_from_db() - self.assertTrue(run1.is_quarantined()) - self.assertTrue(run1_rs.is_quarantined()) - self.assertTrue(run1_outcable.is_quarantined()) - - # Check that all the calls were made correctly. - mock_dataset_quarantine_rcs.assert_called_once_with(step1_orig_output) - mock_execrecord_quarantine_runcomponents.assert_has_calls( - [ - call(run1_rs.execrecord), - call(run1_outcable.execrecord) - ], - any_order=True - ) - - mock_quarantine.assert_has_calls( - [ - call(RunComponent.objects.get(pk=run1_rs.pk), recurse_upward=True, save=True), - call(RunComponent.objects.get(pk=run1_outcable.pk), recurse_upward=True, save=True) - ], - any_order=True - ) - self.assertEquals(mock_quarantine.call_count, 2) - mock_run_quarantine.assert_called_once_with(run1, recurse_upward=True, save=True) - - # These still haven't been mocked yet after being reset. - self.assertFalse(hasattr(mock_start, "assert_not_called")) - self.assertFalse(hasattr(mock_finish_successfully, "assert_not_called")) - self.assertFalse(hasattr(mock_finish_failure, "assert_not_called")) - self.assertFalse(hasattr(mock_begin_recovery, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_pending, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_running, "assert_not_called")) - self.assertFalse(hasattr(mock_decontaminate, "assert_not_called")) - self.assertFalse(hasattr(mock_run_attempt_decontamination, "assert_not_called")) - self.assertFalse(hasattr(mock_execrecord_attempt_decontamination, "assert_not_called")) - self.assertFalse(hasattr(mock_dataset_attempt_decontamination, "assert_not_called")) - self.assertFalse(hasattr(mock_run_start, "assert_not_called")) - self.assertFalse(hasattr(mock_run_stop, "assert_not_called")) - self.assertFalse(hasattr(mock_run_mark_failure, "assert_not_called")) - - mock_dataset_quarantine_rcs.reset_mock() - mock_execrecord_quarantine_runcomponents.reset_mock() - mock_quarantine.reset_mock() - mock_run_quarantine.reset_mock() - - # Now, we try it again. - run2 = Manager.execute_pipeline(self.myUser, pipeline, inputs).get_last_run() - # It should have re-validated run1 and run1_rs. - run1.refresh_from_db() - run1_rs.refresh_from_db() - run1_outcable.refresh_from_db() - self.assertTrue(run1.is_successful()) - self.assertTrue(run1_rs.is_successful()) - self.assertTrue(run1_outcable.is_successful()) - - self.check_run_OK(run2) - run2_step1 = run2.runsteps.first() - run2_outcable = run2.runoutputcables.first() - - # Check all calls. - mock_start.assert_has_calls([ - call(run2_step1), - call(run2_step1.RSICs.first()), - call(run2_outcable) - ]) - self.assertEquals(mock_start.call_count, 3) - - # All of them should have been finished successfully without event. - mock_finish_successfully.assert_has_calls([ - call(run2_step1.RSICs.first(), save=True), - call(run2_step1, save=True), - call(run2_outcable, save=True) - ]) - self.assertEquals(mock_finish_successfully.call_count, 3) - - # Test the decontamination calls. - mock_dataset_attempt_decontamination.assert_called_once_with(step1_orig_output) - mock_execrecord_attempt_decontamination.assert_has_calls( - [ - call(run1_step1.execrecord, step1_orig_output), - call(run1_outcable.execrecord, step1_orig_output) - ], - any_order=True - ) - self.assertEquals(mock_execrecord_attempt_decontamination.call_count, 2) - - mock_decontaminate.assert_has_calls( - [ - call(RunComponent.objects.get(pk=run1_step1.pk), recurse_upward=True, save=True), - call(RunComponent.objects.get(pk=run1_outcable.pk), recurse_upward=True, save=True) - ], - any_order=True - ) - self.assertEquals(mock_decontaminate.call_count, 2) - # run1 attempts to decontaminate itself twice; the first time, it doesn't do anything - # because there's still another RunComponent that's contaminated. - mock_run_attempt_decontamination.assert_has_calls( - [ - call(run1, recurse_upward=True, save=True), - call(run1, recurse_upward=True, save=True) - ], - any_order=True - ) - self.assertEquals(mock_run_attempt_decontamination.call_count, 2) - - # These haven't been mocked. - self.assertFalse(hasattr(mock_dataset_quarantine_rcs, "assert_not_called")) - self.assertFalse(hasattr(mock_execrecord_quarantine_runcomponents, "assert_not_called")) - self.assertFalse(hasattr(mock_quarantine, "assert_not_called")) - self.assertFalse(hasattr(mock_run_quarantine, "assert_not_called")) - self.assertFalse(hasattr(mock_finish_failure, "assert_not_called")) - self.assertFalse(hasattr(mock_begin_recovery, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_pending, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_running, "assert_not_called")) - - @patch.object(Dataset, "attempt_to_decontaminate_runcomponents_using_as_output", autospec=True, - side_effect=Dataset.attempt_to_decontaminate_runcomponents_using_as_output) - @patch.object(Dataset, "quarantine_runcomponents_using_as_output", autospec=True, - side_effect=Dataset.quarantine_runcomponents_using_as_output) - @patch.object(ExecRecord, "attempt_decontamination", autospec=True, - side_effect=ExecRecord.attempt_decontamination) - @patch.object(ExecRecord, "quarantine_runcomponents", autospec=True, - side_effect=ExecRecord.quarantine_runcomponents) - @patch.object(Run, "attempt_decontamination", autospec=True, side_effect=Run.attempt_decontamination) - @patch.object(RunComponent, "decontaminate", autospec=True, side_effect=RunComponent.decontaminate) - @patch.object(Run, "quarantine", autospec=True, side_effect=Run.quarantine) - @patch.object(Run, "mark_failure", autospec=True, side_effect=Run.mark_failure) - @patch.object(Run, "stop", autospec=True, side_effect=Run.stop) - @patch.object(Run, "start", autospec=True, side_effect=Run.start) - @patch.object(RunComponent, "cancel_running", autospec=True, side_effect=RunComponent.cancel_running) - @patch.object(RunComponent, "cancel_pending", autospec=True, side_effect=RunComponent.cancel_pending) - @patch.object(RunComponent, "begin_recovery", autospec=True, side_effect=RunComponent.begin_recovery) - @patch.object(RunComponent, "quarantine", autospec=True, side_effect=RunComponent.quarantine) - @patch.object(RunComponent, "finish_failure", autospec=True, side_effect=RunComponent.finish_failure) - @patch.object(RunComponent, "finish_successfully", autospec=True, side_effect=RunComponent.finish_successfully) - @patch.object(RunComponent, "start", autospec=True, side_effect=RunComponent.start) - def test_execution_decontaminates_quarantined_runcables( - self, - mock_start, - mock_finish_successfully, - mock_finish_failure, - mock_quarantine, - mock_begin_recovery, - mock_cancel_pending, - mock_cancel_running, - mock_run_start, - mock_run_stop, - mock_run_mark_failure, - mock_run_quarantine, - mock_decontaminate, - mock_run_attempt_decontamination, - mock_execrecord_quarantine_runcomponents, - mock_execrecord_attempt_decontamination, - mock_dataset_quarantine_rcs, - mock_dataset_attempt_decontamination - ): - """ - Executing a pipeline with a quarantined runcable properly re-validates it. - """ - # Start by executing a simple one-step pipeline. - pipeline = self.pX - inputs = [self.dataset] - run1 = Manager.execute_pipeline(self.myUser, pipeline, inputs).get_last_run() - self.check_run_OK(run1) - - run1_rs = run1.runsteps.first() - run1_rs_incable = run1_rs.RSICs.first() - run1_outcable = run1.runoutputcables.first() - - # All of the RunComponents should have been started. - mock_start.assert_has_calls([ - call(run1_rs), - call(run1_rs_incable), - call(run1_outcable) - ]) - self.assertEquals(mock_start.call_count, 3) - - # All of them should have been finished successfully without event. - mock_finish_successfully.assert_has_calls([ - call(run1_rs_incable, save=True), - call(run1_rs, save=True), - call(run1_outcable, save=True) - ]) - self.assertEquals(mock_finish_successfully.call_count, 3) - - # These were not called, so have not been mocked yet. - self.assertFalse(hasattr(mock_finish_failure, "assert_not_called")) - self.assertFalse(hasattr(mock_quarantine, "assert_not_called")) - self.assertFalse(hasattr(mock_begin_recovery, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_pending, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_running, "assert_not_called")) - self.assertFalse(hasattr(mock_run_quarantine, "assert_not_called")) - self.assertFalse(hasattr(mock_decontaminate, "assert_not_called")) - self.assertFalse(hasattr(mock_run_attempt_decontamination, "assert_not_called")) - self.assertFalse(hasattr(mock_execrecord_quarantine_runcomponents, "assert_not_called")) - self.assertFalse(hasattr(mock_execrecord_attempt_decontamination, "assert_not_called")) - self.assertFalse(hasattr(mock_dataset_quarantine_rcs, "assert_not_called")) - self.assertFalse(hasattr(mock_dataset_attempt_decontamination, "assert_not_called")) - - mock_run_start.assert_called_once_with(run1, save=True) - mock_run_stop.assert_called_once_with(run1, save=True) - self.assertFalse(hasattr(mock_run_mark_failure, "assert_not_called")) - - mock_run_start.reset_mock() - mock_run_stop.reset_mock() - mock_start.reset_mock() - mock_finish_successfully.reset_mock() - - # Now, let's invalidate the output of the first cable, which is non-trivial. - incable_orig_output = run1_rs_incable.outputs.first() - - corrupted_contents = "Corrupted file" - _, temp_file_path = tempfile.mkstemp() - with open(temp_file_path, "wb") as f: - f.write(corrupted_contents) - incable_orig_output.check_integrity(temp_file_path, self.myUser, notify_all=True) - os.remove(temp_file_path) - - # This should have quarantined run1_rs_incable, and run1, but not run1_rs. - run1.refresh_from_db() - run1_rs.refresh_from_db() - run1_rs_incable.refresh_from_db() - run1_outcable.refresh_from_db() - self.assertTrue(run1.is_quarantined()) - self.assertTrue(run1_rs.is_successful()) - self.assertTrue(run1_rs_incable.is_quarantined()) - - # Check that all the calls were made correctly. - mock_dataset_quarantine_rcs.assert_called_once_with(incable_orig_output) - mock_execrecord_quarantine_runcomponents.assert_called_once_with(run1_rs_incable.execrecord) - - mock_quarantine.assert_called_once_with(RunComponent.objects.get(pk=run1_rs_incable.pk), - recurse_upward=True, save=True) - mock_run_quarantine.assert_called_once_with(run1, recurse_upward=True, save=True) - - # These still haven't been mocked yet after being reset. - self.assertFalse(hasattr(mock_start, "assert_not_called")) - self.assertFalse(hasattr(mock_finish_successfully, "assert_not_called")) - self.assertFalse(hasattr(mock_finish_failure, "assert_not_called")) - self.assertFalse(hasattr(mock_begin_recovery, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_pending, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_running, "assert_not_called")) - self.assertFalse(hasattr(mock_decontaminate, "assert_not_called")) - self.assertFalse(hasattr(mock_run_attempt_decontamination, "assert_not_called")) - self.assertFalse(hasattr(mock_execrecord_attempt_decontamination, "assert_not_called")) - self.assertFalse(hasattr(mock_dataset_attempt_decontamination, "assert_not_called")) - self.assertFalse(hasattr(mock_run_start, "assert_not_called")) - self.assertFalse(hasattr(mock_run_stop, "assert_not_called")) - self.assertFalse(hasattr(mock_run_mark_failure, "assert_not_called")) - - mock_dataset_quarantine_rcs.reset_mock() - mock_execrecord_quarantine_runcomponents.reset_mock() - mock_quarantine.reset_mock() - mock_run_quarantine.reset_mock() - - # Now, we try it again. - run2 = Manager.execute_pipeline(self.myUser, pipeline, inputs).get_last_run() - # It should have re-validated run1 and run1_rs. - run1.refresh_from_db() - run1_rs.refresh_from_db() - run1_rs_incable.refresh_from_db() - self.assertTrue(run1.is_successful()) - self.assertTrue(run1_rs.is_successful()) - self.assertTrue(run1_rs_incable.is_successful()) - - # FIXME note that this is affected by issues #534 and #535; the method does not - # find a suitable ExecRecord. - self.check_run_OK(run2) - run2_step1 = run2.runsteps.first() - run2_outcable = run2.runoutputcables.first() - - # Check all calls. - mock_start.assert_has_calls([ - call(run2_step1), - call(run2_step1.RSICs.first()), - call(run2_outcable) - ]) - self.assertEquals(mock_start.call_count, 3) - - # All of them should have been finished successfully without event. - mock_finish_successfully.assert_has_calls([ - call(run2_step1.RSICs.first(), save=True), - call(run2_step1, save=True), - call(run2_outcable, save=True) - ]) - self.assertEquals(mock_finish_successfully.call_count, 3) - - # Test the decontamination calls. - mock_dataset_attempt_decontamination.assert_called_once_with(incable_orig_output) - mock_execrecord_attempt_decontamination.assert_called_once_with(run1_rs_incable.execrecord, - incable_orig_output) - - mock_decontaminate.assert_called_once_with(RunComponent.objects.get(pk=run1_rs_incable.pk), - recurse_upward=True, save=True) - # run1 attempts to decontaminate itself once because there's only - # one RunComponent contaminated. - mock_run_attempt_decontamination.assert_called_once_with(run1, recurse_upward=True, save=True) - - # These haven't been mocked. - self.assertFalse(hasattr(mock_dataset_quarantine_rcs, "assert_not_called")) - self.assertFalse(hasattr(mock_execrecord_quarantine_runcomponents, "assert_not_called")) - self.assertFalse(hasattr(mock_quarantine, "assert_not_called")) - self.assertFalse(hasattr(mock_run_quarantine, "assert_not_called")) - self.assertFalse(hasattr(mock_finish_failure, "assert_not_called")) - self.assertFalse(hasattr(mock_begin_recovery, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_pending, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_running, "assert_not_called")) - - @patch.object(Dataset, "attempt_to_decontaminate_runcomponents_using_as_output", autospec=True, - side_effect=Dataset.attempt_to_decontaminate_runcomponents_using_as_output) - @patch.object(Dataset, "quarantine_runcomponents_using_as_output", autospec=True, - side_effect=Dataset.quarantine_runcomponents_using_as_output) - @patch.object(ExecRecord, "attempt_decontamination", autospec=True, - side_effect=ExecRecord.attempt_decontamination) - @patch.object(ExecRecord, "quarantine_runcomponents", autospec=True, - side_effect=ExecRecord.quarantine_runcomponents) - @patch.object(Run, "attempt_decontamination", autospec=True, side_effect=Run.attempt_decontamination) - @patch.object(RunComponent, "decontaminate", autospec=True, side_effect=RunComponent.decontaminate) - @patch.object(Run, "quarantine", autospec=True, side_effect=Run.quarantine) - @patch.object(Run, "mark_failure", autospec=True, side_effect=Run.mark_failure) - @patch.object(Run, "stop", autospec=True, side_effect=Run.stop) - @patch.object(Run, "start", autospec=True, side_effect=Run.start) - @patch.object(RunComponent, "cancel_running", autospec=True, side_effect=RunComponent.cancel_running) - @patch.object(RunComponent, "cancel_pending", autospec=True, side_effect=RunComponent.cancel_pending) - @patch.object(RunComponent, "begin_recovery", autospec=True, side_effect=RunComponent.begin_recovery) - @patch.object(RunComponent, "quarantine", autospec=True, side_effect=RunComponent.quarantine) - @patch.object(RunComponent, "finish_failure", autospec=True, side_effect=RunComponent.finish_failure) - @patch.object(RunComponent, "finish_successfully", autospec=True, side_effect=RunComponent.finish_successfully) - @patch.object(RunComponent, "start", autospec=True, side_effect=RunComponent.start) - def test_execution_external_file_decontaminates_quarantined_runcables( - self, - mock_start, - mock_finish_successfully, - mock_finish_failure, - mock_quarantine, - mock_begin_recovery, - mock_cancel_pending, - mock_cancel_running, - mock_run_start, - mock_run_stop, - mock_run_mark_failure, - mock_run_quarantine, - mock_decontaminate, - mock_run_attempt_decontamination, - mock_execrecord_quarantine_runcomponents, - mock_execrecord_attempt_decontamination, - mock_dataset_quarantine_rcs, - mock_dataset_attempt_decontamination - ): - """ - Executing a pipeline on externally-backed data with a quarantined component properly re-validates a RunSIC. - """ - # Set up a working directory and an externally-backed Dataset. - self.working_dir = tempfile.mkdtemp() - self.efd = ExternalFileDirectory( - name="ExecuteTestsEFD", - path=self.working_dir - ) - self.efd.save() - self.ext_path = "ext.txt" - self.full_ext_path = os.path.join(self.working_dir, self.ext_path) - - # Copy the contents of self.dataset to an external file and link the Dataset. - self.raw_dataset.dataset_file.open() - with self.raw_dataset.dataset_file: - with open(self.full_ext_path, "wb") as f: - f.write(self.raw_dataset.dataset_file.read()) - - # Create a new externally-backed Dataset. - externally_backed_ds = Dataset.create_dataset( - self.full_ext_path, - user=self.myUser, - keep_file=False, - name="ExternallyBackedDS", - description="Dataset with external data and no internal data", - externalfiledirectory=self.efd - ) - - # Start by executing a simple one-step pipeline. - pipeline = self.pX_raw - inputs = [externally_backed_ds] - run1 = Manager.execute_pipeline(self.myUser, pipeline, inputs).get_last_run() - self.check_run_OK(run1) - - run1_rs = run1.runsteps.first() - run1_rs_incable = run1_rs.RSICs.first() - run1_outcable = run1.runoutputcables.first() - - # All of the RunComponents should have been started. - mock_start.assert_has_calls([ - call(run1_rs), - call(run1_rs_incable), - call(run1_outcable) - ]) - self.assertEquals(mock_start.call_count, 3) - - # All of them should have been finished successfully without event. - mock_finish_successfully.assert_has_calls([ - call(run1_rs_incable, save=True), - call(run1_rs, save=True), - call(run1_outcable, save=True) - ]) - self.assertEquals(mock_finish_successfully.call_count, 3) - - # These were not called, so have not been mocked yet. - self.assertFalse(hasattr(mock_finish_failure, "assert_not_called")) - self.assertFalse(hasattr(mock_quarantine, "assert_not_called")) - self.assertFalse(hasattr(mock_begin_recovery, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_pending, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_running, "assert_not_called")) - self.assertFalse(hasattr(mock_run_quarantine, "assert_not_called")) - self.assertFalse(hasattr(mock_decontaminate, "assert_not_called")) - self.assertFalse(hasattr(mock_run_attempt_decontamination, "assert_not_called")) - self.assertFalse(hasattr(mock_execrecord_quarantine_runcomponents, "assert_not_called")) - self.assertFalse(hasattr(mock_execrecord_attempt_decontamination, "assert_not_called")) - self.assertFalse(hasattr(mock_dataset_quarantine_rcs, "assert_not_called")) - self.assertFalse(hasattr(mock_dataset_attempt_decontamination, "assert_not_called")) - - mock_run_start.assert_called_once_with(run1, save=True) - mock_run_stop.assert_called_once_with(run1, save=True) - self.assertFalse(hasattr(mock_run_mark_failure, "assert_not_called")) - - mock_run_start.reset_mock() - mock_run_stop.reset_mock() - mock_start.reset_mock() - mock_finish_successfully.reset_mock() - - # Now, let's corrupt the input. - corrupted_contents = "Corrupted file" - with open(self.full_ext_path, "wb") as f: - f.write(corrupted_contents) - externally_backed_ds.check_integrity(self.full_ext_path, self.myUser, notify_all=True) - - # This should have quarantined run1_rs_incable and run1 but not run1_rs. - run1.refresh_from_db() - run1_rs.refresh_from_db() - run1_rs_incable.refresh_from_db() - self.assertTrue(run1.is_quarantined()) - self.assertTrue(run1_rs.is_successful()) - self.assertTrue(run1_rs_incable.is_quarantined()) - - # Check that all the calls were made correctly. - mock_dataset_quarantine_rcs.assert_called_once_with(externally_backed_ds) - mock_execrecord_quarantine_runcomponents.assert_called_once_with(run1_rs_incable.execrecord) - - mock_quarantine.assert_called_once_with(RunComponent.objects.get(pk=run1_rs_incable.pk), - recurse_upward=True, save=True) - mock_run_quarantine.assert_called_once_with(run1, recurse_upward=True, save=True) - - # These still haven't been mocked yet after being reset. - self.assertFalse(hasattr(mock_start, "assert_not_called")) - self.assertFalse(hasattr(mock_finish_successfully, "assert_not_called")) - self.assertFalse(hasattr(mock_finish_failure, "assert_not_called")) - self.assertFalse(hasattr(mock_begin_recovery, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_pending, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_running, "assert_not_called")) - self.assertFalse(hasattr(mock_decontaminate, "assert_not_called")) - self.assertFalse(hasattr(mock_run_attempt_decontamination, "assert_not_called")) - self.assertFalse(hasattr(mock_execrecord_attempt_decontamination, "assert_not_called")) - self.assertFalse(hasattr(mock_dataset_attempt_decontamination, "assert_not_called")) - self.assertFalse(hasattr(mock_run_start, "assert_not_called")) - self.assertFalse(hasattr(mock_run_stop, "assert_not_called")) - self.assertFalse(hasattr(mock_run_mark_failure, "assert_not_called")) - - mock_dataset_quarantine_rcs.reset_mock() - mock_execrecord_quarantine_runcomponents.reset_mock() - mock_quarantine.reset_mock() - mock_run_quarantine.reset_mock() - - # Now we fix the contents of the Dataset. - self.raw_dataset.dataset_file.open() - with self.raw_dataset.dataset_file: - with open(self.full_ext_path, "wb") as f: - f.write(self.raw_dataset.dataset_file.read()) - - # Now, we try it again. - run2 = Manager.execute_pipeline(self.myUser, pipeline, inputs).get_last_run() - # It should have re-validated run1 and run1_rs. - run1.refresh_from_db() - run1_rs.refresh_from_db() - run1_rs_incable.refresh_from_db() - self.assertTrue(run1.is_successful()) - self.assertTrue(run1_rs.is_successful()) - self.assertTrue(run1_rs_incable.is_successful()) - - self.check_run_OK(run2) - run2_step1 = run2.runsteps.first() - run2_outcable = run2.runoutputcables.first() - - # Check all calls. - mock_start.assert_has_calls([ - call(run2_step1), - call(run2_step1.RSICs.first()), - call(run2_outcable) - ]) - self.assertEquals(mock_start.call_count, 3) - - # All of them should have been finished successfully without event. - mock_finish_successfully.assert_has_calls([ - call(run2_step1.RSICs.first(), save=True), - call(run2_step1, save=True), - call(run2_outcable, save=True) - ]) - self.assertEquals(mock_finish_successfully.call_count, 3) - - # Test the decontamination calls. - mock_dataset_attempt_decontamination.assert_called_once_with(externally_backed_ds) - mock_execrecord_attempt_decontamination.assert_called_once_with(run1_rs_incable.execrecord, - externally_backed_ds) - - mock_decontaminate.assert_called_once_with(RunComponent.objects.get(pk=run1_rs_incable.pk), - recurse_upward=True, save=True) - # run1 attempts to decontaminate itself once because there's only - # one RunComponent contaminated. - mock_run_attempt_decontamination.assert_called_once_with(run1, recurse_upward=True, save=True) - - # These haven't been mocked. - self.assertFalse(hasattr(mock_dataset_quarantine_rcs, "assert_not_called")) - self.assertFalse(hasattr(mock_execrecord_quarantine_runcomponents, "assert_not_called")) - self.assertFalse(hasattr(mock_quarantine, "assert_not_called")) - self.assertFalse(hasattr(mock_run_quarantine, "assert_not_called")) - self.assertFalse(hasattr(mock_finish_failure, "assert_not_called")) - self.assertFalse(hasattr(mock_begin_recovery, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_pending, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_running, "assert_not_called")) - - shutil.rmtree(self.working_dir) - - @patch.object(Dataset, "attempt_to_decontaminate_runcomponents_using_as_output", autospec=True, - side_effect=Dataset.attempt_to_decontaminate_runcomponents_using_as_output) - @patch.object(Dataset, "quarantine_runcomponents_using_as_output", autospec=True, - side_effect=Dataset.quarantine_runcomponents_using_as_output) - @patch.object(ExecRecord, "attempt_decontamination", autospec=True, - side_effect=ExecRecord.attempt_decontamination) - @patch.object(ExecRecord, "quarantine_runcomponents", autospec=True, - side_effect=ExecRecord.quarantine_runcomponents) - @patch.object(Run, "attempt_decontamination", autospec=True, side_effect=Run.attempt_decontamination) - @patch.object(RunComponent, "decontaminate", autospec=True, side_effect=RunComponent.decontaminate) - @patch.object(Run, "quarantine", autospec=True, side_effect=Run.quarantine) - @patch.object(Run, "mark_failure", autospec=True, side_effect=Run.mark_failure) - @patch.object(Run, "stop", autospec=True, side_effect=Run.stop) - @patch.object(Run, "start", autospec=True, side_effect=Run.start) - @patch.object(RunComponent, "cancel_running", autospec=True, side_effect=RunComponent.cancel_running) - @patch.object(RunComponent, "cancel_pending", autospec=True, side_effect=RunComponent.cancel_pending) - @patch.object(RunComponent, "begin_recovery", autospec=True, side_effect=RunComponent.begin_recovery) - @patch.object(RunComponent, "quarantine", autospec=True, side_effect=RunComponent.quarantine) - @patch.object(RunComponent, "finish_failure", autospec=True, side_effect=RunComponent.finish_failure) - @patch.object(RunComponent, "finish_successfully", autospec=True, side_effect=RunComponent.finish_successfully) - @patch.object(RunComponent, "start", autospec=True, side_effect=RunComponent.start) - def test_execution_external_file_decontaminates_quarantined_runsteps( - self, - mock_start, - mock_finish_successfully, - mock_finish_failure, - mock_quarantine, - mock_begin_recovery, - mock_cancel_pending, - mock_cancel_running, - mock_run_start, - mock_run_stop, - mock_run_mark_failure, - mock_run_quarantine, - mock_decontaminate, - mock_run_attempt_decontamination, - mock_execrecord_quarantine_runcomponents, - mock_execrecord_attempt_decontamination, - mock_dataset_quarantine_rcs, - mock_dataset_attempt_decontamination - ): - """ - Executing a pipeline on externally-backed data with a quarantined component properly re-validates RunSteps. - """ - # Set up a working directory and an externally-backed Dataset. - self.working_dir = tempfile.mkdtemp() - self.efd = ExternalFileDirectory( - name="ExecuteTestsEFD", - path=self.working_dir - ) - self.efd.save() - self.ext_path = "ext.txt" - self.full_ext_path = os.path.join(self.working_dir, self.ext_path) - - # Copy the contents of self.dataset to an external file and link the Dataset. - self.raw_dataset.dataset_file.open() - with self.raw_dataset.dataset_file: - with open(self.full_ext_path, "wb") as f: - f.write(self.raw_dataset.dataset_file.read()) - - # Create a new externally-backed Dataset. - externally_backed_ds = Dataset.create_dataset( - self.full_ext_path, - user=self.myUser, - keep_file=False, - name="ExternallyBackedDS", - description="Dataset with external data and no internal data", - externalfiledirectory=self.efd - ) - - # Start by executing a simple one-step pipeline. - pipeline = self.pX_raw - inputs = [externally_backed_ds] - run1 = Manager.execute_pipeline(self.myUser, pipeline, inputs).get_last_run() - self.check_run_OK(run1) - - run1_rs = run1.runsteps.first() - run1_rs_incable = run1_rs.RSICs.first() - run1_outcable = run1.runoutputcables.first() - - # All of the RunComponents should have been started. - mock_start.assert_has_calls([ - call(run1_rs), - call(run1_rs_incable), - call(run1_outcable) - ]) - self.assertEquals(mock_start.call_count, 3) - - # All of them should have been finished successfully without event. - mock_finish_successfully.assert_has_calls([ - call(run1_rs_incable, save=True), - call(run1_rs, save=True), - call(run1_outcable, save=True) - ]) - self.assertEquals(mock_finish_successfully.call_count, 3) - - # These were not called, so have not been mocked yet. - self.assertFalse(hasattr(mock_finish_failure, "assert_not_called")) - self.assertFalse(hasattr(mock_quarantine, "assert_not_called")) - self.assertFalse(hasattr(mock_begin_recovery, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_pending, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_running, "assert_not_called")) - self.assertFalse(hasattr(mock_run_quarantine, "assert_not_called")) - self.assertFalse(hasattr(mock_decontaminate, "assert_not_called")) - self.assertFalse(hasattr(mock_run_attempt_decontamination, "assert_not_called")) - self.assertFalse(hasattr(mock_execrecord_quarantine_runcomponents, "assert_not_called")) - self.assertFalse(hasattr(mock_execrecord_attempt_decontamination, "assert_not_called")) - self.assertFalse(hasattr(mock_dataset_quarantine_rcs, "assert_not_called")) - self.assertFalse(hasattr(mock_dataset_attempt_decontamination, "assert_not_called")) - - mock_run_start.assert_called_once_with(run1, save=True) - mock_run_stop.assert_called_once_with(run1, save=True) - self.assertFalse(hasattr(mock_run_mark_failure, "assert_not_called")) - - mock_run_start.reset_mock() - mock_run_stop.reset_mock() - mock_start.reset_mock() - mock_finish_successfully.reset_mock() - - # Now, let's corrupt the input. - corrupted_contents = "Corrupted file" - with open(self.full_ext_path, "wb") as f: - f.write(corrupted_contents) - - # We also delete the output of the step so that it's forced to run. - output_ds = run1_rs.outputs.first() - output_ds.dataset_file.delete(save=True) - - # We do another run. - run2 = Manager.execute_pipeline(self.myUser, pipeline, inputs).get_last_run() - - # This should have quarantined run1_rs_incable and run1 but not run1_rs. - run1.refresh_from_db() - run1_rs.refresh_from_db() - run1_rs_incable.refresh_from_db() - self.assertTrue(run1.is_quarantined()) - self.assertTrue(run1_rs.is_successful()) - self.assertTrue(run1_rs_incable.is_quarantined()) - - run2_rs = run2.runsteps.first() - run2_rs_incable = run2_rs.RSICs.first() - self.assertTrue(run2.is_failed()) - self.assertTrue(run2_rs.is_cancelled()) - self.assertTrue(run2_rs_incable.is_cancelled()) - - # The step and incable should have been started. - mock_start.assert_has_calls([ - call(run2_rs), - call(run2_rs_incable), - ]) - self.assertEquals(mock_start.call_count, 2) - - # Both are cancelled. - mock_cancel_running.assert_has_calls([ - call(run2_rs_incable, save=True), - call(run2_rs, save=True) - ]) - self.assertEquals(mock_cancel_running.call_count, 2) - - # The run started, was marked a failure, and stopped. - mock_run_start.assert_called_once_with(run2, save=True) - mock_run_mark_failure.assert_called_once_with(run2, save=True) - mock_run_stop.assert_called_once_with(run2, save=True) - - # The input dataset should have called quarantine. - mock_dataset_quarantine_rcs.assert_called_once_with(externally_backed_ds) - mock_execrecord_quarantine_runcomponents.assert_called_once_with(run2_rs_incable.execrecord) - mock_quarantine.assert_called_once_with( - RunComponent.objects.get(pk=run1_rs_incable.pk), recurse_upward=True, save=True - ) - mock_run_quarantine.assert_called_once_with(run1, recurse_upward=True, save=True) - - # These were not called, so have not been mocked yet. - self.assertFalse(hasattr(mock_finish_successfully, "assert_not_called")) - self.assertFalse(hasattr(mock_finish_failure, "assert_not_called")) - self.assertFalse(hasattr(mock_begin_recovery, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_pending, "assert_not_called")) - self.assertFalse(hasattr(mock_run_quarantine, "assert_not_called")) - self.assertFalse(hasattr(mock_decontaminate, "assert_not_called")) - self.assertFalse(hasattr(mock_run_attempt_decontamination, "assert_not_called")) - self.assertFalse(hasattr(mock_execrecord_attempt_decontamination, "assert_not_called")) - self.assertFalse(hasattr(mock_dataset_attempt_decontamination, "assert_not_called")) - - mock_run_start.reset_mock() - mock_run_stop.reset_mock() - mock_run_mark_failure.reset_mock() - mock_start.reset_mock() - mock_finish_successfully.reset_mock() - mock_dataset_quarantine_rcs.reset_mock() - mock_execrecord_quarantine_runcomponents.reset_mock() - mock_quarantine.reset_mock() - mock_run_quarantine.reset_mock() - - # Now we fix the contents of the Dataset. - self.raw_dataset.dataset_file.open() - with self.raw_dataset.dataset_file: - with open(self.full_ext_path, "wb") as f: - f.write(self.raw_dataset.dataset_file.read()) - - # Now, we try it again. - run3 = Manager.execute_pipeline(self.myUser, pipeline, inputs).get_last_run() - # It should have re-validated run1 and run1_rs. - run1.refresh_from_db() - run1_rs.refresh_from_db() - run1_rs_incable.refresh_from_db() - self.assertTrue(run1.is_successful()) - self.assertTrue(run1_rs.is_successful()) - self.assertTrue(run1_rs_incable.is_successful()) - - # The stuff from run2 should be left alone. - run2.refresh_from_db() - run2_rs.refresh_from_db() - run2_rs_incable.refresh_from_db() - self.assertTrue(run2.is_failed()) - self.assertTrue(run2_rs.is_cancelled()) - self.assertTrue(run2_rs_incable.is_cancelled()) - - self.check_run_OK(run3) - run3_rs = run3.runsteps.first() - run3_rs_incable = run3_rs.RSICs.first() - run3_outcable = run3.runoutputcables.first() - - # Check all calls. - mock_start.assert_has_calls([ - call(run3_rs), - call(run3_rs_incable), - call(run3_outcable) - ]) - self.assertEquals(mock_start.call_count, 3) - - # All of them should have been finished successfully without event. - mock_finish_successfully.assert_has_calls([ - call(run3_rs_incable, save=True), - call(run3_rs, save=True), - call(run3_outcable, save=True) - ]) - self.assertEquals(mock_finish_successfully.call_count, 3) - - # Test the decontamination calls. - mock_dataset_attempt_decontamination.assert_called_once_with(externally_backed_ds) - mock_execrecord_attempt_decontamination.assert_called_once_with(run1_rs_incable.execrecord, - externally_backed_ds) - - mock_decontaminate.assert_called_once_with(RunComponent.objects.get(pk=run1_rs_incable.pk), - recurse_upward=True, save=True) - # run1 attempts to decontaminate itself once. - mock_run_attempt_decontamination.assert_called_once_with(run1, recurse_upward=True, save=True) - - # These haven't been mocked. - self.assertFalse(hasattr(mock_dataset_quarantine_rcs, "assert_not_called")) - self.assertFalse(hasattr(mock_execrecord_quarantine_runcomponents, "assert_not_called")) - self.assertFalse(hasattr(mock_quarantine, "assert_not_called")) - self.assertFalse(hasattr(mock_run_quarantine, "assert_not_called")) - self.assertFalse(hasattr(mock_finish_failure, "assert_not_called")) - self.assertFalse(hasattr(mock_begin_recovery, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_pending, "assert_not_called")) - self.assertFalse(hasattr(mock_cancel_running, "assert_not_called")) - - shutil.rmtree(self.working_dir) - - -class ExecuteSandboxPathWithSpacesTests(ExecuteTestsBase): - - def setUp(self): - ExecuteTestsBase.setUp(self) - self.sandbox_path_original = settings.SANDBOX_PATH - settings.SANDBOX_PATH = "Sandbox path with spaces" - - def tearDown(self): - ExecuteTestsBase.tearDown(self) - settings.SANDBOX_PATH = self.sandbox_path_original - - def test_pipeline_sandbox_path_has_spaces(self): - """Execute a Pipeline in a sandbox that has spaces in the path.""" - pipeline = self.find_nonraw_pipeline(self.myUser) - inputs = self.find_inputs_for_pipeline(pipeline) - self.assertTrue(all(i.is_OK() for i in inputs)) - self.assertFalse(all(i.is_raw() for i in inputs)) - run = Manager.execute_pipeline(self.myUser, pipeline, inputs).get_last_run() - - self.check_run_OK(run) - - # Check the full is_complete and is_successful. - self.assertTrue(run.is_complete()) - self.assertTrue(run.is_successful()) - - self.assertIsNone(run.clean()) - self.assertIsNone(run.complete_clean()) - - -class SandboxTests(ExecuteTestsBase): - - def test_sandbox_no_input(self): - """ - A Sandbox cannot be created if the pipeline has inputs but none are supplied. - """ - p = Pipeline(family=self.pf, revision_name="blah", revision_desc="blah blah", user=self.myUser) - p.save() - p.create_input(compounddatatype=self.pX_in_cdt, dataset_name="in", dataset_idx=1) - self.assertRaisesRegexp(ValueError, - re.escape('Pipeline "{}" expects 1 inputs, but 0 were supplied'.format(p)), - lambda: Manager.execute_pipeline(self.myUser, p, [])) - - def test_sandbox_too_many_inputs(self): - """ - A Sandbox cannot be created if the pipeline has fewer inputs than are supplied. - """ - p = Pipeline(family=self.pf, revision_name="blah", revision_desc="blah blah", user=self.myUser) - p.save() - self.assertRaisesRegexp(ValueError, - re.escape('Pipeline "{}" expects 0 inputs, but 1 were supplied'.format(p)), - lambda: Manager.execute_pipeline(self.myUser, p, [self.dataset])) - - def test_sandbox_correct_inputs(self): - """ - We can create a Sandbox if the supplied inputs match the pipeline inputs. - """ - p = Pipeline(family=self.pf, revision_name="blah", revision_desc="blah blah", user=self.myUser) - p.save() - p.create_input(compounddatatype=self.pX_in_cdt, - dataset_name="in", - dataset_idx=1, - min_row=8, - max_row=12) - # Assert no ValueError raised. - Manager.execute_pipeline(self.myUser, p, [self.dataset]) - - def test_sandbox_raw_expected_nonraw_supplied(self): - """ - Can't create a Sandbox if the pipeline expects raw input and we give it nonraw. - """ - p = Pipeline(family=self.pf, revision_name="blah", revision_desc="blah blah", user=self.myUser) - p.save() - p.create_input(dataset_name="in", dataset_idx=1) - self.assertRaisesRegexp( - ValueError, - re.escape('Pipeline "{}" expected input {} to be raw, but got one with ' - 'CompoundDatatype "{}"'.format(p, 1, self.dataset.structure.compounddatatype)), - lambda: Manager.execute_pipeline(self.myUser, p, [self.dataset]) - ) - - def test_sandbox_nonraw_expected_raw_supplied(self): - """ - Can't create a Sandbox if the pipeline expects non-raw input and we give it raw. - """ - p = Pipeline(family=self.pf, revision_name="blah", revision_desc="blah blah", user=self.myUser) - p.save() - p.create_input(compounddatatype=self.pX_in_cdt, dataset_name="in", dataset_idx=1) - tf = tempfile.NamedTemporaryFile(delete=False) - tf.write("foo") - tf.close() - raw_dataset = Dataset.create_dataset( - tf.name, - user=self.myUser, - name="foo", - description="bar" - ) - self.assertRaisesRegexp( - ValueError, - re.escape('Pipeline "{}" expected input {} to be of CompoundDatatype "{}", but got raw' - .format(p, 1, self.pX_in_cdt)), - lambda: Manager.execute_pipeline(self.myUser, p, [raw_dataset]) - ) - os.remove(tf.name) - - def test_sandbox_cdt_mismatch(self): - """ - Can't create a Sandbox if the pipeline expects an input with one CDT - and we give it the wrong one. - """ - p = Pipeline(family=self.pf, revision_name="blah", revision_desc="blah blah", user=self.myUser) - p.save() - p.create_input(compounddatatype=self.mA_in_cdt, dataset_name="in", dataset_idx=1) - self.assertRaisesRegexp(ValueError, - re.escape('Pipeline "{}" expected input {} to be of CompoundDatatype "{}", but got one ' - 'with CompoundDatatype "{}"' - .format(p, 1, self.mA_in_cdt, self.dataset.structure.compounddatatype)), - lambda: Manager.execute_pipeline(self.myUser, p, [self.dataset])) - - def test_sandbox_too_many_rows(self): - """ - Can't create a Sandbox if the pipeline expects an input with one CDT - and we give it the wrong one. - """ - p = Pipeline(family=self.pf, revision_name="blah", revision_desc="blah blah", user=self.myUser) - p.save() - p.create_input(compounddatatype=self.pX_in_cdt, - dataset_name="in", - dataset_idx=1, - min_row=2, - max_row=4) - expected_message = ( - 'Pipeline "{}" expected input {} to have between {} and {} rows, ' - 'but got one with {}'.format(p, 1, 2, 4, self.dataset.num_rows())) - self.assertRaisesRegexp(ValueError, - re.escape(expected_message), - lambda: Manager.execute_pipeline(self.myUser, p, [self.dataset])) - - def test_sandbox_too_few_rows(self): - """ - Can't create a Sandbox if the pipeline expects an input with one CDT - and we give it the wrong one. - """ - p = Pipeline(family=self.pf, revision_name="blah", revision_desc="blah blah", user=self.myUser) - p.save() - p.create_input(compounddatatype=self.pX_in_cdt, - dataset_name="in", - dataset_idx=1, - min_row=20) - expected_message = ( - 'Pipeline "{}" expected input {} to have between {} and {} rows, ' - 'but got one with {}'.format(p, 1, 20, sys.maxint, self.dataset.num_rows())) - self.assertRaisesRegexp(ValueError, - re.escape(expected_message), - lambda: Manager.execute_pipeline(self.myUser, p, [self.dataset])) - - -@skipIfDBFeature('is_mocked') -class RestoreReusableDatasetTest(TestCase): - """ - Scenario where an output is marked as reusable, and it needs to be restored. - - There are three methods: - * sums_and_products - take each row of two integers, calculate sum and - product, then shuffle all the result rows. This makes it reusable, but not - deterministic. - * total_sums - copy the first row, then one more row with the sum of all - the sums from the remaining rows. - * total_products - copy the first row, then one more row with the sum of all - the products from the remaining rows. - - """ - fixtures = ["restore_reusable_dataset"] - - def setUp(self): - install_fixture_files("restore_reusable_dataset") - - def tearDown(self): - remove_fixture_files() - - def test_load_run_plan(self): - pipeline = Pipeline.objects.get(revision_name='sums only') - dataset = Dataset.objects.get(name='pairs') - - run = Run(user=pipeline.user, pipeline=pipeline) - run.save() - run.inputs.create(dataset=dataset, index=1) - - sandbox = Sandbox(run) - run_plan = RunPlan() - run_plan.load(sandbox.run, sandbox.inputs) - step_plans = run_plan.step_plans - - self.assertEqual(sandbox.run, run_plan.run) - self.assertEqual([StepPlan(1), StepPlan(2)], step_plans) - self.assertEqual([DatasetPlan(dataset)], - step_plans[0].inputs) - self.assertEqual([DatasetPlan(step_num=1, output_num=1)], - step_plans[0].outputs) - self.assertEqual(step_plans[0].outputs, step_plans[1].inputs) - self.assertIs(step_plans[0].outputs[0], step_plans[1].inputs[0]) - - self.assertEqual([DatasetPlan(dataset)], run_plan.inputs) - self.assertEqual([DatasetPlan(step_num=2, output_num=1)], - run_plan.outputs) - - def test_find_consistent_execution_for_rerun(self): - pipeline = Pipeline.objects.get(revision_name='sums only') - input_dataset = Dataset.objects.get(name='pairs') - step1_output_dataset = Dataset.objects.get(id=2) - step2_output_dataset = Dataset.objects.get(id=3) - - run = Run(user=pipeline.user, pipeline=pipeline) - run.save() - run.inputs.create(dataset=input_dataset, index=1) - - sandbox = Sandbox(run) - run_plan = RunPlan() - run_plan.load(sandbox.run, sandbox.inputs) - - run_plan.find_consistent_execution() - step_plans = run_plan.step_plans - - self.assertEqual([DatasetPlan(step1_output_dataset)], - step_plans[0].outputs) - self.assertEqual([DatasetPlan(step2_output_dataset)], - step_plans[1].outputs) - - def test_find_consistent_execution_for_new_run(self): - pipeline = Pipeline.objects.get(revision_name='sums and products') - input_dataset = Dataset.objects.get(name='pairs') - - run = Run(user=pipeline.user, pipeline=pipeline) - run.save() - run.inputs.create(dataset=input_dataset, index=1) - - sandbox = Sandbox(run) - - run_plan = RunPlan() - run_plan.load(sandbox.run, sandbox.inputs) - - run_plan.find_consistent_execution() - step_plans = run_plan.step_plans - - self.assertEqual([DatasetPlan(step_num=1, output_num=1)], - step_plans[0].outputs) - self.assertEqual([DatasetPlan(step_num=2, output_num=1)], - step_plans[1].outputs) - self.assertEqual([DatasetPlan(step_num=3, output_num=1)], - step_plans[2].outputs) - - def test_find_consistent_execution_with_missing_output(self): - pipeline = Pipeline.objects.get(revision_name='sums only') - pipeline.steps.get(step_num=1).outputs_to_delete.clear() - - input_dataset = Dataset.objects.get(name='pairs') - - run = Run(user=pipeline.user, pipeline=pipeline) - run.save() - run.inputs.create(dataset=input_dataset, index=1) - - sandbox = Sandbox(run) - - run_plan = RunPlan() - run_plan.load(sandbox.run, sandbox.inputs) - - run_plan.find_consistent_execution() - step_plans = run_plan.step_plans - - self.assertIsNone(step_plans[0].execrecord) - - def test_find_consistent_execution_with_missing_output_of_deterministic_method(self): - pipeline = Pipeline.objects.get(revision_name='sums only') - pipeline_step = pipeline.steps.get(step_num=1) - pipeline_step.outputs_to_delete.clear() - method = pipeline_step.transformation.definite - method.reusable = Method.DETERMINISTIC - method.save() - method.clean() - - input_dataset = Dataset.objects.get(name='pairs') - - run = Run(user=pipeline.user, pipeline=pipeline) - run.save() - run.inputs.create(dataset=input_dataset, index=1) - - sandbox = Sandbox(run) - - run_plan = RunPlan() - run_plan.load(sandbox.run, sandbox.inputs) - - run_plan.find_consistent_execution() - step_plans = run_plan.step_plans - - self.assertIsNotNone(step_plans[0].execrecord) - - -class ExecuteExternalInputTests(ExecuteTestsBase): - - def setUp(self): - super(ExecuteExternalInputTests, self).setUp() - - self.working_dir = tempfile.mkdtemp() - self.efd = ExternalFileDirectory( - name="ExecuteTestsEFD", - path=self.working_dir - ) - self.efd.save() - self.ext_path = "ext.txt" - self.full_ext_path = os.path.join(self.working_dir, self.ext_path) - - def tearDown(self): - super(ExecuteExternalInputTests, self).tearDown() - shutil.rmtree(self.working_dir) - - def test_pipeline_external_file_input(self): - """Execution of a pipeline whose input is externally-backed.""" - - # Copy the contents of self.dataset to an external file and link the Dataset. - self.raw_dataset.dataset_file.open() - with self.raw_dataset.dataset_file: - with open(self.full_ext_path, "wb") as f: - f.write(self.raw_dataset.dataset_file.read()) - - # Create a new externally-backed Dataset. - external_ds = Dataset.create_dataset( - self.full_ext_path, - user=self.myUser, - keep_file=False, - name="ExternalDS", - description="Dataset with external data and no internal data", - externalfiledirectory=self.efd - ) - - # Execute pipeline - run = Manager.execute_pipeline(self.myUser, self.pX_raw, [external_ds]).get_last_run() - - self.check_run_OK(run) - - def test_pipeline_external_file_input_deleted(self): - """Execution of a pipeline whose input is missing.""" - - # Copy the contents of self.dataset to an external file and link the Dataset. - self.raw_dataset.dataset_file.open() - with self.raw_dataset.dataset_file: - with open(self.full_ext_path, "wb") as f: - f.write(self.raw_dataset.dataset_file.read()) - - # Create a new externally-backed Dataset. - external_missing_ds = Dataset.create_dataset( - self.full_ext_path, - user=self.myUser, - keep_file=False, - name="ExternalMissingDS", - description="Dataset with missing external data and no internal data", - externalfiledirectory=self.efd - ) - # Remove the external file. - os.remove(self.full_ext_path) - - # Execute pipeline - run = Manager.execute_pipeline(self.myUser, self.pX_raw, [external_missing_ds]).get_last_run() - # 2017-07-11, issue #661: the Manager no longer attempts to run a pipeline with a missing - # input -- get_last_run() will return None in this case. - self.assertTrue(run is None) - # The run should be cancelled by the first cable. - # self.assertTrue(run.is_cancelled()) - # rsic = run.runsteps.get(pipelinestep__step_num=1).RSICs.first() - # self.assertTrue(rsic.is_cancelled()) - # self.assertTrue(hasattr(rsic, "input_integrity_check")) - # self.assertTrue(rsic.input_integrity_check.read_failed) - - def test_pipeline_external_file_input_corrupted(self): - """Execution of a pipeline whose input is corrupted.""" - - # Copy the contents of self.dataset to an external file and link the Dataset. - self.raw_dataset.dataset_file.open() - with self.raw_dataset.dataset_file: - with open(self.full_ext_path, "wb") as f: - f.write(self.raw_dataset.dataset_file.read()) - - # Create a new externally-backed Dataset. - external_corrupted_ds = Dataset.create_dataset( - self.full_ext_path, - user=self.myUser, - keep_file=False, - name="ExternalCorruptedDS", - description="Dataset with corrupted external data and no internal data", - externalfiledirectory=self.efd - ) - # Tamper with the external file. - with open(self.full_ext_path, "wb") as f: - f.write("Corrupted") - - # Execute pipeline - run = Manager.execute_pipeline(self.myUser, self.pX_raw, [external_corrupted_ds]).get_last_run() - - # The run should fail on the first cable. - self.assertFalse(run.is_successful()) - rsic = run.runsteps.get(pipelinestep__step_num=1).RSICs.first() - self.assertFalse(rsic.is_successful()) - self.assertTrue(hasattr(rsic, "input_integrity_check")) - self.assertTrue(rsic.input_integrity_check.is_md5_conflict()) diff --git a/kive/sandbox/tests_rm.py b/kive/sandbox/tests_rm.py deleted file mode 100644 index 89c302a57..000000000 --- a/kive/sandbox/tests_rm.py +++ /dev/null @@ -1,613 +0,0 @@ -from unittest import skip, skipIf - -from django.test import TestCase, skipIfDBFeature -from django.utils import timezone -from django.contrib.auth.models import User -from django.conf import settings - -import tempfile -import shutil -import os.path -import time - -from librarian.models import Dataset -import kive.testing_utils as tools -from pipeline.models import Pipeline, PipelineFamily -from kive.tests import install_fixture_files, remove_fixture_files, BaseTestCases -from method.models import Method -from fleet.workers import Manager -from archive.models import Run -from fleet.slurmlib import DummySlurmScheduler -from fleet.dockerlib import DummyDockerHandler, DockerHandler -import file_access_utils - - -@skipIfDBFeature('is_mocked') -class SandboxRMTestCase(BaseTestCases.SlurmExecutionTestCase): - def setUp(self): - tools.create_sandbox_testing_tools_environment(self) - - def tearDown(self): - tools.destroy_sandbox_testing_tools_environment(self) - - -@skipIfDBFeature('is_mocked') -class ExecuteResultTestsRM(TestCase): - """ - Tests on the results of executing Pipelines. - """ - fixtures = ["execute_result_tests_rm"] - - def setUp(self): - install_fixture_files("execute_result_tests_rm") - self.method_complement = Method.objects.get( - family__name="DNA complement", - revision_name="v1" - ) - - self.pipeline_complement = Pipeline.objects.get( - family__name="DNA complement", - revision_name="v1" - ) - self.pipeline_reverse = Pipeline.objects.get( - family__name="DNA reverse", - revision_name="v1" - ) - self.pipeline_revcomp = Pipeline.objects.get( - family__name="DNA revcomp", - revision_name="v1" - ) - - self.user_alice = User.objects.get(username="alice") - - self.comp_run = self.pipeline_complement.pipeline_instances.order_by("start_time").first() - self.comp_run_2 = self.pipeline_complement.pipeline_instances.order_by("start_time").last() - self.reverse_run = self.pipeline_reverse.pipeline_instances.first() - self.revcomp_run = self.pipeline_revcomp.pipeline_instances.first() - - self.dataset_labdata = Dataset.objects.get( - name="lab data", - user=self.user_alice - ) - - # Tracking down CDTs is a pain.... - self.cdt_record = self.method_complement.inputs.first().structure.compounddatatype - - def tearDown(self): - tools.clean_up_all_files() - remove_fixture_files() - - def test_execute_pipeline_run(self): - """ - Check the coherence of Runs created when a pipeline is executed the first time. - """ - run = self.comp_run - self.assertEqual(run.user, self.user_alice) - self.assertEqual(run.start_time < timezone.now(), True) - self.assertEqual(run.is_complete(), True) - self.assertEqual(run.parent_runstep, None) - self.assertEqual(run.complete_clean(), None) - - def test_execute_pipeline_runstep(self): - """ - Check the coherence of a RunStep created when a Pipeline is executed the first time. - """ - run = self.comp_run - # sandbox_complement has only one step, so this is OK. - runstep = run.runsteps.first() - - self.assertEqual(runstep.run, run) - self.assertTrue(runstep.start_time < timezone.now()) - self.assertFalse(runstep.reused) - self.assertTrue(runstep.is_complete()) - self.assertEqual(runstep.complete_clean(), None) - self.assertFalse(hasattr(runstep, "child_run")) - self.assertTrue(runstep.is_successful()) - self.assertEqual(runstep.outputs.count(), 1) - - def test_execute_pipeline_dataset_contents(self): - """ - Test that the content checks, which take place as part of Pipeline - execution, pass in the ordinary Pipeline execution case. - """ - run = self.comp_run - runstep = run.runsteps.first() - execrecord = runstep.execrecord - dataset = execrecord.execrecordouts.first().dataset - check = dataset.content_checks.first() - - self.assertEqual(dataset.content_checks.count(), 1) # should have been checked once - self.assertEqual(check.dataset, dataset) - self.assertEqual(check.end_time is None, False) - self.assertEqual(check.start_time <= check.end_time, True) - self.assertEqual(check.start_time.date(), check.end_time.date()) - self.assertEqual(check.is_fail(), False) - - @skip(reason="Fails because Method.submit_code has been removed for docker" - " support. Needs to be fixed or removed.") - def test_execute_pipeline_dataset(self): - """ - Test the integrity of a Dataset output by a PipelineStep in - the middle of a Pipeline. - NOTE: 2017-10-03: this tests fails because Method.submit_code has been removed - for docker support. - """ - # Figure out the MD5 of the output file created when the complement method - # is run on Alice's data to check against the result of the run. - tmpdir = tempfile.mkdtemp(dir=file_access_utils.sandbox_base_path()) - file_access_utils.configure_sandbox_permissions(tmpdir) - outfile = os.path.join(tmpdir, "output") - stdout_path = os.path.join(tmpdir, "stdout.txt") - stderr_path = os.path.join(tmpdir, "stderr.txt") - - self.method_complement.install(tmpdir) - - # Set up the dummy scheduler. - slurm_sched_class = DummySlurmScheduler - slurm_sched_class.slurm_is_alive() - - complement_job_handle = self.method_complement.submit_code( - tmpdir, - [self.dataset_labdata.dataset_file.file.name], - [outfile], - stdout_path, - stderr_path, - slurm_sched_class=slurm_sched_class - ) - - is_done = False - while not is_done: - time.sleep(settings.DEFAULT_SLURM_CHECK_INTERVAL) - accounting_info = DummySlurmScheduler.get_accounting_info([complement_job_handle]) - if len(accounting_info) > 0: - curr_state = accounting_info[complement_job_handle.job_id]["state"] - is_done = curr_state == DummySlurmScheduler.COMPLETED - - slurm_sched_class.shutdown() - - labdata_compd_md5 = file_access_utils.compute_md5(open(outfile)) - shutil.rmtree(tmpdir) - - run = self.comp_run - runstep = run.runsteps.first() - execrecord = runstep.execrecord - dataset = execrecord.execrecordouts.first().dataset - ds = runstep.outputs.first() - - self.assertEqual(dataset.MD5_checksum, labdata_compd_md5) - self.assertEqual(dataset, ds) - self.assertEqual(hasattr(dataset, "usurps"), False) - self.assertEqual(dataset.has_data(), True) - self.assertEqual(dataset.num_rows(), 10) - self.assertEqual(dataset.is_raw(), False) - self.assertEqual(dataset.get_cdt(), self.cdt_record) - self.assertEqual(dataset.structure.compounddatatype, self.cdt_record) - self.assertEqual(dataset.structure.num_rows, 10) - self.assertEqual(dataset.is_OK(), True) - - def test_execute_pipeline_runstep_execrecordout(self): - """ - Check the coherence of a RunStep's ExecRecord's ExecRecordOut, created - when a Pipeline is executed the first time. - """ - run = self.comp_run - - pipelinestep = self.pipeline_complement.steps.first() # 1 step - runstep = run.runsteps.first() - dataset_out = runstep.outputs.first() - execlog = runstep.log - execrecord = runstep.execrecord - execrecordout = execrecord.execrecordouts.first() - - self.assertEqual(execrecordout is None, False) - self.assertEqual(execrecordout.execrecord, execrecord) - self.assertEqual(execrecordout.dataset, dataset_out) - self.assertEqual(execrecordout.generic_output.definite, pipelinestep.transformation.outputs.first()) - self.assertEqual(execrecordout.has_data(), True) - self.assertEqual(execrecordout.is_OK(), True) - self.assertNotEqual(None, execlog) - - def test_execute_pipeline_runstep_execrecord(self): - """ - Check the coherence of a RunStep's ExecRecord, created when a Pipeline - is executed the first time. - """ - run = self.comp_run - runstep = run.runsteps.first() - execlog = runstep.log - execrecord = runstep.execrecord - outputs = self.method_complement.outputs.all() - - self.assertEqual(execrecord.generator, execlog) - self.assertEqual(execrecord.complete_clean(), None) - self.assertEqual(execrecord.general_transf(), runstep.pipelinestep.transformation.method) - self.assertEqual(execrecord.provides_outputs(outputs), True) - self.assertEqual(execrecord.outputs_OK(), True) - - def test_execute_pipeline_reuse(self): - """ - An identical pipeline, run in a different sandbox, should reuse an ExecRecord - and not create an ExecLog. - """ - step1 = self.comp_run.runsteps.first() - step2 = self.comp_run_2.runsteps.first() - - self.assertEqual(step1.reused, False) - self.assertEqual(step2.reused, True) - self.assertFalse(step2.has_log()) - self.assertEqual(step1.execrecord, step2.execrecord) - - def test_execute_pipeline_fill_in_ER(self): - """ - Running an identical Pipeline where we did not keep the data around the first time - should fill in an existing ExecRecord, but also create a new ExecLog. - """ - step1 = self.comp_run.runsteps.first() - step2 = self.comp_run_2.runsteps.first() - - self.assertEqual(step1.reused, False) - self.assertEqual(step2.reused, True) - self.assertTrue(step1.has_log()) - self.assertFalse(step2.has_log()) - self.assertEqual(step1.execrecord, step2.execrecord) - - def test_execute_pipeline_reuse_within_different_pipeline(self): - """ - Running the same dataset through the same Method, in two different - pipelines, should reuse an ExecRecord. - """ - step1 = self.reverse_run.runsteps.first() # 1 step - step2 = self.revcomp_run.runsteps.get(pipelinestep__step_num=1) - - self.assertEqual(step1.reused, False) - self.assertEqual(step2.reused, True) - self.assertFalse(step2.has_log()) - self.assertEqual(step1.execrecord, step2.execrecord) - - def test_execute_pipeline_output_dataset(self): - """ - A Pipeline with no deleted outputs should have a Dataset as an output. - """ - output = self.comp_run.runoutputcables.first() - output_dataset = output.execrecord.execrecordouts.first().dataset - self.assertEqual(output_dataset is not None, True) - - def test_trivial_cable_num_rows(self): - """ - A trivial cable should have the same dataset all the way through. - """ - step = self.comp_run.runsteps.first() - step_output_dataset = step.execrecord.execrecordouts.first().dataset - - outcable = self.comp_run.runoutputcables.first() - outcable_input_dataset = outcable.execrecord.execrecordins.first().dataset - outcable_output_dataset = outcable.execrecord.execrecordouts.first().dataset - - self.assertEqual(step_output_dataset, outcable_input_dataset) - self.assertEqual(outcable_input_dataset, outcable_output_dataset) - self.assertEqual(step_output_dataset.num_rows(), outcable_input_dataset.num_rows()) - self.assertEqual(outcable_input_dataset.num_rows(), outcable_output_dataset.num_rows()) - - def test_execute_pipeline_num_rows(self): - """ - A pipeline which does not change the number of rows in a dataset, - should have the same number of rows in all datasets along the way. - """ - incable = self.comp_run.runsteps.first().RSICs.first() - incable_input_dataset = incable.execrecord.execrecordins.first().dataset - incable_output_dataset = incable.execrecord.execrecordins.first().dataset - - step = self.comp_run.runsteps.first() - step_input_dataset = step.execrecord.execrecordins.first().dataset - step_output_dataset = step.execrecord.execrecordouts.first().dataset - - outcable = self.comp_run.runoutputcables.first() - outcable_input_dataset = outcable.execrecord.execrecordins.first().dataset - outcable_output_dataset = outcable.execrecord.execrecordouts.first().dataset - - self.assertEqual(incable_input_dataset.num_rows(), self.dataset_labdata.num_rows()) - self.assertEqual(incable_input_dataset.num_rows(), incable_output_dataset.num_rows()) - self.assertEqual(incable_output_dataset.num_rows(), step_input_dataset.num_rows()) - self.assertEqual(step_input_dataset.num_rows(), step_output_dataset.num_rows()) - self.assertEqual(step_output_dataset.num_rows(), outcable_input_dataset.num_rows()) - self.assertEqual(outcable_input_dataset.num_rows(), outcable_output_dataset.num_rows()) - - -@skipIfDBFeature('is_mocked') -class ExecuteDiscardedIntermediateTests(BaseTestCases.SlurmExecutionTestCase): - fixtures = ["execute_discarded_intermediate_tests_rm"] - - def setUp(self): - install_fixture_files("execute_discarded_intermediate_tests_rm") - self.revcomp_pf = PipelineFamily.objects.get(name="DNA revcomp") - self.pipeline_revcomp_v2 = self.revcomp_pf.members.get(revision_name="2") - self.pipeline_revcomp_v3 = self.revcomp_pf.members.get(revision_name="3") - - self.user_alice = User.objects.get(username="alice") - - self.revcomp_v2_run = self.pipeline_revcomp_v2.pipeline_instances.first() # only one exists - - self.dataset_labdata = Dataset.objects.get( - name="lab data", - user=self.user_alice - ) - - def tearDown(self): - tools.clean_up_all_files() - remove_fixture_files() - - def test_discard_intermediate_file(self): - """ - A Pipeline which indicates one of its intermediate outputs should not be kept, - should not create any datasets for that output. - """ - runstep = self.revcomp_v2_run.runsteps.get(pipelinestep__step_num=1) - output = runstep.execrecord.execrecordouts.first().dataset - step = self.pipeline_revcomp_v2.steps.get(step_num=1) - self.assertEqual(runstep.pipelinestep.outputs_to_retain(), []) - self.assertEqual(output.has_data(), False) - self.assertNotEqual(None, step) - - def test_recover_intermediate_dataset(self): - """ - Test recovery of an intermediate dataset. - """ - # In the fixture, we already ran self.pipeline_revcomp_v2, which discards the intermediate - # output. We now run v3, which will recover it. - run = Manager.execute_pipeline( - self.user_alice, - self.pipeline_revcomp_v3, - [self.dataset_labdata], - docker_handler_class=DummyDockerHandler - ).get_last_run() - - self.assertTrue(run.is_successful()) - - -class BadRunTestsBase(object): - """ - Foundations of tests for when things go wrong during Pipeline execution. - - We split this code out into an object (not a TestCase) so it can be - reused in another test class. - """ - def setUp(self): - tools.create_grandpa_sandbox_environment(self) - - def tearDown(self): - tools.destroy_grandpa_sandbox_environment(self) - - def cable_tester(self, runstep): - for rsic in runstep.RSICs.all(): - self.assertTrue(rsic.is_successful()) - - def test_method_fails(self, - slurm_sched_class=DummySlurmScheduler, - docker_handler_class=DummyDockerHandler): - """Properly handle a failed method in a pipeline.""" - run = Manager.execute_pipeline( - self.user_grandpa, - self.pipeline_fubar, - [self.dataset_grandpa], - slurm_sched_class=slurm_sched_class, - docker_handler_class=docker_handler_class - ).get_last_run() - - self.assertTrue(run.is_failed()) - self.assertIsNone(run.complete_clean()) - - runstep1 = run.runsteps.get(pipelinestep__step_num=1) - self.cable_tester(runstep1) - self.assertIsNone(runstep1.complete_clean()) - self.assertTrue(runstep1.is_successful()) - - runstep2 = run.runsteps.get(pipelinestep__step_num=2) - self.cable_tester(runstep2) - self.assertIsNone(runstep2.complete_clean()) - self.assertTrue(runstep2.is_failed()) - - log = runstep2.log - - self.assertFalse(log.is_successful()) - self.assertEqual(log.methodoutput.return_code, 1) - self.assertEqual(log.missing_outputs(), [runstep2.execrecord.execrecordouts.first().dataset]) - - -@skipIfDBFeature('is_mocked') -class BadRunTests(BaseTestCases.SlurmExecutionTestCase, BadRunTestsBase): - """ - Tests for when things go wrong during Pipeline execution. - """ - def setUp(self): - BaseTestCases.SlurmExecutionTestCase.setUp(self) - BadRunTestsBase.setUp(self) - - def tearDown(self): - BaseTestCases.SlurmExecutionTestCase.tearDown(self) - BadRunTestsBase.tearDown(self) - - pass - - -@skipIfDBFeature('is_mocked') -class FindDatasetTests(BaseTestCases.SlurmExecutionTestCase): - """ - Tests for first_generator_of_dataset. - """ - fixtures = ['find_datasets'] - - def setUp(self): - install_fixture_files('find_datasets') - - def tearDown(self): - remove_fixture_files() - - def test_find_dataset_pipeline_input_and_step_output(self): - """ - Finding a Dataset which was input to a Pipeline should return None - as the generator, and the top-level run as the run. - - Finding a Dataset which was output from a step, and also input - to a cable, should return the step (and in particular, not the cable). - """ - self.pipeline_noop = Pipeline.objects.get(family__name="simple pipeline") - self.dataset_words = Dataset.objects.get(name='blahblah') - self.user_bob = User.objects.get(username='bob') - - mgr = Manager.execute_pipeline(self.user_bob, - self.pipeline_noop, - [self.dataset_words], - docker_handler_class=DummyDockerHandler) - x = mgr.history_queue.pop() - self.assertIsNone(x.run.complete_clean()) - self.assertTrue(x.run.is_successful()) - - run, gen = x.first_generator_of_dataset(self.dataset_words) - self.assertEqual(run, x.run) - self.assertEqual(gen, None) - - dataset_out_intermediate = x.run.runsteps.first().execrecord.execrecordouts.first().dataset - run_2, gen_2 = x.first_generator_of_dataset(dataset_out_intermediate) - self.assertEqual(run_2, x.run) - self.assertEqual(gen_2, self.pipeline_noop.steps.first()) - - def test_find_dataset_pipeline_input_and_intermediate_custom_wire(self): - """ - Finding a Dataset which was passed through a custom wire to a - Pipeline should return the cable as the generator, and the top-level - run as the run. - - Finding a Dataset which was produced by a custom wire as an - intermediate step should return the cable as the generator, and the - top-level run as the run. - """ - self.pipeline_twostep = Pipeline.objects.get(family__name="two-step pipeline") - self.dataset_backwords = Dataset.objects.get(name='backwords') - self.user_bob = User.objects.get(username='bob') - - mgr = Manager.execute_pipeline(self.user_bob, - self.pipeline_twostep, - [self.dataset_backwords], - docker_handler_class=DummyDockerHandler) - sandbox = mgr.history_queue.pop() - self.assertIsNone(sandbox.run.complete_clean()) - self.assertTrue(sandbox.run.is_successful()) - - runcable = sandbox.run.runsteps.get(pipelinestep__step_num=1).RSICs.first() - dataset_to_find = runcable.execrecord.execrecordouts.first().dataset - - run, gen = sandbox.first_generator_of_dataset(dataset_to_find) - self.assertEqual(run, sandbox.run) - self.assertEqual(gen, runcable.PSIC) - - # Testing on an intermediate Dataset. - runcable_2 = sandbox.run.runsteps.get(pipelinestep__step_num=2).RSICs.first() - dataset_to_find_2 = runcable_2.execrecord.execrecordouts.first().dataset - - run_2, gen_2 = sandbox.first_generator_of_dataset(dataset_to_find_2) - self.assertEqual(run_2, sandbox.run) - self.assertEqual(gen_2, runcable_2.PSIC) - - def test_find_dataset_subpipeline_input_and_intermediate(self): - """ - Find a dataset in a sub-pipeline, which is output from a step. - - Find a dataset in a sub-pipeline, which is input to the sub-pipeline - on a custom cable. - """ - self.pipeline_nested = Pipeline.objects.get(family__name="nested pipeline") - self.dataset_backwords = Dataset.objects.get(name='backwords') - self.user_bob = User.objects.get(username='bob') - - mgr = Manager.execute_pipeline(self.user_bob, - self.pipeline_nested, - [self.dataset_backwords], - docker_handler_class=DummyDockerHandler) - sandbox = mgr.history_queue.pop() - self.assertIsNone(sandbox.run.complete_clean()) - self.assertTrue(sandbox.run.is_successful()) - - subpipeline_step = sandbox.run.runsteps.get(pipelinestep__step_num=2) - subrun = subpipeline_step.child_run - runstep = subrun.runsteps.first() - outrecord = runstep.execrecord.execrecordouts.first() - dataset_to_find = outrecord.dataset - - run, gen = sandbox.first_generator_of_dataset(dataset_to_find) - self.assertEqual(run, subrun) - self.assertEqual(gen, runstep.pipelinestep) - - cable = runstep.RSICs.first() - dataset_to_find_2 = runstep.execrecord.execrecordins.first().dataset - - run_2, gen_2 = sandbox.first_generator_of_dataset(dataset_to_find_2) - self.assertEqual(run_2, subrun) - self.assertEqual(gen_2, cable.PSIC) - - -class RawTests(SandboxRMTestCase): - - def setUp(self): - super(RawTests, self).setUp() - - self.addTypeEqualityFunc(str, self.assertMultiLineEqual) - self.pipeline_raw = tools.make_first_pipeline( - "raw noop", "a pipeline to do nothing to raw data", - self.user_bob) - tools.create_linear_pipeline(self.pipeline_raw, [self.method_noop_raw], "raw_in", "raw_out") - self.pipeline_raw.create_outputs() - - self.dataset_raw = Dataset.create_dataset( - "/usr/share/dict/words", - user=self.user_bob, - cdt=None, - keep_file=True, - name="raw", - description="some raw data" - ) - - def test_execute_pipeline_raw(self): - """Execute a raw Pipeline.""" - run = Manager.execute_pipeline(self.user_bob, - self.pipeline_raw, - [self.dataset_raw], - docker_handler_class=DummyDockerHandler).get_last_run() - run.refresh_from_db() - self.assertTrue(run.is_successful()) - - def test_execute_pipeline_raw_twice(self): - """Execute a raw Pipeline and reuse an ExecRecord.""" - run = Manager.execute_pipeline(self.user_bob, - self.pipeline_raw, - [self.dataset_raw], - docker_handler_class=DummyDockerHandler).get_last_run() - run = Run.objects.get(pk=run.pk) - self.assertTrue(run.is_successful()) - - run2 = Manager.execute_pipeline(self.user_bob, - self.pipeline_raw, - [self.dataset_raw], - docker_handler_class=DummyDockerHandler).get_last_run() - run2 = Run.objects.get(pk=run2.pk) - self.assertTrue(run2.is_successful()) - - @skipIf(not settings.RUN_DOCKER_TESTS, "Docker tests disabled.") - def test_execute_pipeline_raw_with_docker(self): - """Execute a raw Pipeline.""" - self.maxDiff = None - run = Manager.execute_pipeline(self.user_bob, - self.pipeline_raw, - [self.dataset_raw], - docker_handler_class=DockerHandler).get_last_run() - run.refresh_from_db() - stderr_path = os.path.join(run.sandbox_path, - "step1", - "logs", - "step1_stderr_slurmID0_node0.txt") - with open(stderr_path, 'rU') as f: - stderr_text = f.read() - self.assertEqual("", stderr_text) - self.assertTrue(run.is_successful()) - - def tearDown(self): - super(RawTests, self).tearDown() diff --git a/kive/stopwatch/tests.py b/kive/stopwatch/tests.py index 34fd869e7..df649a296 100644 --- a/kive/stopwatch/tests.py +++ b/kive/stopwatch/tests.py @@ -8,19 +8,17 @@ from django.core.exceptions import ValidationError import kive.testing_utils as tools -from pipeline.models import PipelineFamily +from container.models import ContainerRun @skipIfDBFeature('is_mocked') class StopwatchTests(TestCase): - fixtures = ["em_sandbox_test_environment"] + fixtures = ["container_run"] # The fixture creates self.pE_run, which is a # Stopwatch. We'll use this as our Stopwatch. def setUp(self): - self.pf = PipelineFamily.objects.get(name="Pipeline_family") - self.pE = self.pf.members.get(revision_name="pE_name") - self.pE_run = self.pE.pipeline_instances.first() + self.pE_run = ContainerRun.objects.first() def tearDown(self): tools.clean_up_all_files() @@ -118,4 +116,4 @@ def test_stop(self): self.pE_run.start(clean=False) self.assertFalse(self.pE_run.has_ended()) self.pE_run.stop(clean=False) - self.assertTrue(self.pE_run.has_ended()) \ No newline at end of file + self.assertTrue(self.pE_run.has_ended()) diff --git a/utils/prepare_initial_pipeline.py b/utils/prepare_initial_pipeline.py deleted file mode 100755 index 7ea634c8b..000000000 --- a/utils/prepare_initial_pipeline.py +++ /dev/null @@ -1,66 +0,0 @@ -#! /usr/bin/env python - -# Prepare a test Pipeline and a dataset to run it with for the purpose of generating a fixture. -# Make sure DJANGO_SETTINGS_MODULE is set appropriately in the shell. -# Dump data using -# ./manage.py dumpdata --indent=4 librarian method pipeline transformation archive > [filename] - -from django.core.files import File -from django.contrib.auth.models import User - -import metadata.models -from librarian.models import Dataset -import method.models -import kive.testing_utils as tools - -# This comes from the initial_user fixture. -kive_user = User.objects.get(pk=1) - -test_fasta = Dataset.create_dataset( - file_path="../samplecode/step_0_raw.fasta", - user=kive_user, - cdt=None, - keep_file=True, - name="TestFASTA", - description="Toy FASTA file for testing pipelines" -) - -# Set up a test Pipeline. -resource = method.models.CodeResource(name="Fasta2CSV", description="FASTA converter script", filename="Fasta2CSV.py") -resource.clean() -resource.save() -with open("../samplecode/fasta2csv.py", "rb") as f: - revision = method.models.CodeResourceRevision( - coderesource=resource, - revision_name="v1", - revision_desc="First version", - content_file=File(f)) - revision.clean() - revision.save() -resource.clean() - -# The CDT to use is defined in the initial_data fixture. -fasta_CSV_CDT = metadata.models.CompoundDatatype.objects.get(pk=4) -fasta_to_CSV = tools.make_first_method("Fasta2CSV", "Converts FASTA to CSV", revision) -fasta_to_CSV.create_input(compounddatatype=None, dataset_name="FASTA", dataset_idx=1) -fasta_to_CSV.create_output(compounddatatype=fasta_CSV_CDT, dataset_name="CSV", dataset_idx=1) - -test_pipeline = tools.make_first_pipeline("Fasta2CSV", "One-step pipeline wrapper for Fasta2CSV Method") -tools.create_linear_pipeline(test_pipeline, [fasta_to_CSV], "pipeline_input", "pipeline_output") - -# Set the positions so this looks OK in the interface. These numbers were taken from actually -# laying this out in the interface previously. -pi = test_pipeline.inputs.first() -pi.x = 0.07 -pi.y = 0.194 -pi.save() - -po = test_pipeline.outputs.first() -po.x = 0.85 -po.y = 0.194 -po.save() - -step1 = test_pipeline.steps.get(step_num=1) -step1.x = 0.15 -step1.y = 0.21 -step1.save()