From 6ac0e2b4591ed2ed12f6ac2c605d5ca75833ffad Mon Sep 17 00:00:00 2001 From: joezuntz Date: Thu, 10 Oct 2024 05:58:19 -0700 Subject: [PATCH 1/2] Update registry syntax for v1 --- ceci/pipeline.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ceci/pipeline.py b/ceci/pipeline.py index fe85798..da50970 100644 --- a/ceci/pipeline.py +++ b/ceci/pipeline.py @@ -528,14 +528,16 @@ def data_registry_lookup(self, info): #pragma: no cover results = registry.Query.find_datasets(["dataset.dataset_id"], [filter]) # Check that we find exactly one dataset matching the query - results = list(results) - if len(results) == 0: + if not results: raise ValueError(f"Could not find any dataset matching {info} in registry") - elif len(results) > 1: + + results = results['dataset.dataset_id'] + + if len(results) > 1: raise ValueError(f"Found multiple datasets matching {info} in registry") # Get the absolute path - return registry.Query.get_dataset_absolute_path(results[0].dataset_id) + return registry.Query.get_dataset_absolute_path(results[0]) def process_overall_inputs(self, inputs): From 04e97ac142e0353813807d159c759b766e6d13d3 Mon Sep 17 00:00:00 2001 From: joezuntz Date: Fri, 11 Oct 2024 05:59:47 -0700 Subject: [PATCH 2/2] require only valid files in data registry --- ceci/pipeline.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ceci/pipeline.py b/ceci/pipeline.py index da50970..f8b6c80 100644 --- a/ceci/pipeline.py +++ b/ceci/pipeline.py @@ -524,8 +524,11 @@ def data_registry_lookup(self, info): #pragma: no cover else: raise ValueError("Must specify either id or name in registry lookup") + # Whatever the lookup, we always require a dataset which has not beed deleted or replaced + status_filter = registry.Query.gen_filter("dataset.status", "==", 1) + # Main finder method - results = registry.Query.find_datasets(["dataset.dataset_id"], [filter]) + results = registry.Query.find_datasets(["dataset.dataset_id"], [filter, status_filter]) # Check that we find exactly one dataset matching the query if not results: