From 21726478dc752029b3cb909a8083e85c0cd26001 Mon Sep 17 00:00:00 2001 From: Eric Charles Date: Mon, 17 Jan 2022 11:01:43 -0800 Subject: [PATCH] Joe's last few requests, deal with else-no-return, added items() and value() overrides to StageConfig --- .pylintrc | 2 +- ceci/config.py | 12 ++++++++++-- ceci/sites/ccin2p3.py | 22 +++++++++++----------- ceci/sites/cori.py | 21 +++++++++++---------- ceci/sites/local.py | 21 +++++++++++---------- nb/ceci_interactive_example.ipynb | 10 +++++++++- tests/test_interactive.py | 6 ++++++ 7 files changed, 59 insertions(+), 35 deletions(-) diff --git a/.pylintrc b/.pylintrc index 2b130ae..e8ef0f6 100644 --- a/.pylintrc +++ b/.pylintrc @@ -60,7 +60,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,abstract-method,broad-except,invalid-name,line-too-long,wrong-import-order,wrong-import-position,too-many-statements,super-with-arguments,import-outside-toplevel,too-many-arguments,too-many-instance-attributes,unspecified-encoding +disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,abstract-method,broad-except,invalid-name,line-too-long,wrong-import-order,wrong-import-position,too-many-statements,super-with-arguments,import-outside-toplevel,too-many-arguments,too-many-instance-attributes,unspecified-encoding,no-else-return [REPORTS] diff --git a/ceci/config.py b/ceci/config.py index fff4e03..27968de 100644 --- a/ceci/config.py +++ b/ceci/config.py @@ -141,7 +141,7 @@ def __init__(self, **kwargs): def __str__(self): """ Override __str__ casting to deal with `StageParameter` object in the map """ s = "{" - for key, attr in self.items(): + for key, attr in dict.items(self): if isinstance(attr, StageParameter): val = attr.value else: @@ -180,6 +180,14 @@ def __setattr__(self, key, value): """ Allow attribute-like parameter setting """ return self.__setitem__(key, value) + def items(self): + """ Override items() to get the parameters values instead of the objects """ + return [(key, cast_to_streamable(value)) for key, value in dict.items(self)] + + def values(self): + """ Override items() to get the parameters values instead of the objects """ + return [cast_to_streamable(value) for value in dict.values(self)] + def set_config(self, input_config, args): """ Utility function to load configuration @@ -216,7 +224,7 @@ def set_config(self, input_config, args): def reset(self): """ Reset values to their defaults """ - for _, val in self.items(): + for _, val in dict.items(self): if isinstance(val, StageParameter): val.set_to_default() diff --git a/ceci/sites/ccin2p3.py b/ceci/sites/ccin2p3.py index 9b3dd22..a0d781a 100644 --- a/ceci/sites/ccin2p3.py +++ b/ceci/sites/ccin2p3.py @@ -56,17 +56,17 @@ def command(self, cmd, sec): f"{cmd} {mpi2} " f"{bash_end}" ) - - # In the non-container case this is much easier - paths_env = ( - "PYTHONPATH=" + (":".join(paths)) + ":$PYTHONPATH" if paths else "" - ) - return ( - f"OMP_NUM_THREADS={sec.threads_per_process} " - f"{paths_env} " - f"{mpi1} " - f"{cmd} {mpi2}" - ) + else: + # In the non-container case this is much easier + paths_env = ( + "PYTHONPATH=" + (":".join(paths)) + ":$PYTHONPATH" if paths else "" + ) + return ( + f"OMP_NUM_THREADS={sec.threads_per_process} " + f"{paths_env} " + f"{mpi1} " + f"{cmd} {mpi2}" + ) def configure_for_parsl(self): #pylint: disable=no-self-use """Utility function to set parsl configuration parameters""" diff --git a/ceci/sites/cori.py b/ceci/sites/cori.py index 4f90254..308c7f9 100644 --- a/ceci/sites/cori.py +++ b/ceci/sites/cori.py @@ -73,16 +73,17 @@ def command(self, cmd, sec): f"{cmd} {mpi2} " f"{paths_end} " ) - paths_env = ( - ("PYTHONPATH=" + (":".join(paths)) + ":$PYTHONPATH") if paths else "" - ) - return ( - # In the non-container case this is much easier - f"OMP_NUM_THREADS={sec.threads_per_process} " - f"{paths_env} " - f"{mpi1} " - f"{cmd} {mpi2}" - ) + else: + paths_env = ( + ("PYTHONPATH=" + (":".join(paths)) + ":$PYTHONPATH") if paths else "" + ) + return ( + # In the non-container case this is much easier + f"OMP_NUM_THREADS={sec.threads_per_process} " + f"{paths_env} " + f"{mpi1} " + f"{cmd} {mpi2}" + ) def configure_for_mini(self): """Utility function to setup self for local execution""" diff --git a/ceci/sites/local.py b/ceci/sites/local.py index 7b12e89..4fe63dc 100644 --- a/ceci/sites/local.py +++ b/ceci/sites/local.py @@ -55,16 +55,17 @@ def command(self, cmd, sec): f"{cmd} {mpi2} " f"{paths_end}" ) - # In the non-container case this is much easier - paths_env = ( - "PYTHONPATH=" + (":".join(paths)) + ":$PYTHONPATH" if paths else "" - ) - return ( - f"OMP_NUM_THREADS={sec.threads_per_process} " - f"{paths_env} " - f"{mpi1} " - f"{cmd} {mpi2}" - ) + else: + # In the non-container case this is much easier + paths_env = ( + "PYTHONPATH=" + (":".join(paths)) + ":$PYTHONPATH" if paths else "" + ) + return ( + f"OMP_NUM_THREADS={sec.threads_per_process} " + f"{paths_env} " + f"{mpi1} " + f"{cmd} {mpi2}" + ) def configure_for_parsl(self): """Utility function to set parsl configuration parameters""" diff --git a/nb/ceci_interactive_example.ipynb b/nb/ceci_interactive_example.ipynb index 981f50b..7fe7f3f 100644 --- a/nb/ceci_interactive_example.ipynb +++ b/nb/ceci_interactive_example.ipynb @@ -229,7 +229,7 @@ "metadata": {}, "outputs": [], "source": [ - "pipe2.load_configs(overall_inputs, pipeline.run_config, pipeline.stages_config)" + "pipe2.initialize(overall_inputs, pipeline.run_config, pipeline.stages_config)" ] }, { @@ -280,6 +280,14 @@ "metadata": {}, "outputs": [], "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4b969da", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/tests/test_interactive.py b/tests/test_interactive.py index b6f1b25..751251f 100644 --- a/tests/test_interactive.py +++ b/tests/test_interactive.py @@ -47,6 +47,12 @@ def test_config(): assert config.chunk_rows == 5000 assert config.get_type('chunk_rows') == int + + values = config.values() + for key, value in config.items(): + #assert value == config[key].value + assert value in values + def test_interactive_pipeline():