From 85f8e68271d8e65f3470152c5036eccee63cfb2c Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Sun, 6 Dec 2020 20:12:49 -0600 Subject: [PATCH 01/34] Use napoleon to render numpydoc style. --- doc/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/conf.py b/doc/conf.py index 798a4d0ed..f22e28146 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -45,6 +45,7 @@ def __getattr__(cls, name): "sphinx.ext.autodoc", "sphinx.ext.autosummary", "sphinx.ext.intersphinx", + "sphinx.ext.napoleon", "sphinx.ext.viewcode", ] From 93fbae508fdd5d5ffd1429b30b735dccff3d34c5 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Sun, 6 Dec 2020 20:34:09 -0600 Subject: [PATCH 02/34] NumPy-doc-ify aggregates. --- flow/aggregates.py | 171 +++++++++++++++++++++++++++------------------ 1 file changed, 104 insertions(+), 67 deletions(-) diff --git a/flow/aggregates.py b/flow/aggregates.py index 3363e81c0..8c0508cfe 100644 --- a/flow/aggregates.py +++ b/flow/aggregates.py @@ -15,8 +15,14 @@ class aggregator: """Decorator for operation functions that operate on aggregates. - By default, if the ``aggregator_function`` is ``None``, - an aggregate of all jobs will be created. + By default, if the ``aggregator_function`` is ``None``, an aggregate of all + jobs will be created. + + Example + ------- + + The code block below defines a :class:`~.FlowOperation` that prints the + total length of the provided aggregate of jobs. .. code-block:: python @@ -25,27 +31,22 @@ class aggregator: def foo(*jobs): print(len(jobs)) - :param aggregator_function: + Parameters + ---------- + aggregator_function : callable or None A callable that performs aggregation of jobs. It takes in a list of jobs and can return or yield subsets of jobs as an iterable. The default behavior is creating a single aggregate of all jobs. - :type aggregator_function: - callable or None - :param sort_by: + sort_by : str or None Before aggregating, sort the jobs by a given statepoint parameter. The default behavior is no sorting. - :type sort_by: - str or None - :param sort_ascending: + sort_ascending : bool States if the jobs are to be sorted in ascending order. The default value is True. - :type sort_ascending: - bool - :param select: + select : callable or None Condition for filtering individual jobs. This is passed as the callable argument to :func:`filter`. The default behavior is no filtering. - :type select: - callable or None + """ def __init__( @@ -87,8 +88,10 @@ def groupsof(cls, num=1, sort_by=None, sort_ascending=True, select=None): project and they are aggregated in groups of 3, then the generated aggregates will have lengths 3, 3, 3, and 1. - The code block below provides an example of how jobs can be aggregated - in groups of 2. + Example + ------- + + The code block below shows how to aggregate jobs in groups of 2. .. code-block:: python @@ -97,26 +100,26 @@ def groupsof(cls, num=1, sort_by=None, sort_ascending=True, select=None): def foo(*jobs): print(len(jobs)) - :param num: + Parameters + ---------- + num : int The default size of aggregates excluding the final aggregate. - :type num: - int - :param sort_by: + sort_by : str or None Before aggregating, sort the jobs by a given statepoint parameter. The default behavior is no sorting. - :type sort_by: - str or None - :param sort_ascending: + sort_ascending : bool States if the jobs are to be sorted in ascending order. The default value is True. - :type sort_ascending: - bool - :param select: + select : callable or None Condition for filtering individual jobs. This is passed as the callable argument to :func:`filter`. The default behavior is no filtering. - :type select: - callable or None + + Returns + ------- + aggregator : :class:`~.aggregator` + The :meth:`~.groupsof` aggregator. + """ try: if num != int(num): @@ -144,7 +147,10 @@ def aggregator_function(jobs): def groupby(cls, key, default=None, sort_by=None, sort_ascending=True, select=None): """Aggregate jobs according to matching state point values. - The below code block provides an example of how to aggregate jobs + Example + ------- + + The code block below provides an example of how to aggregate jobs having a common state point parameter ``'sp'`` whose value, if not found, is replaced by a default value of -1. @@ -155,34 +161,32 @@ def groupby(cls, key, default=None, sort_by=None, sort_ascending=True, select=No def foo(*jobs): print(len(jobs)) - :param key: + Parameters + ---------- + key : str, Iterable, or callable The method by which jobs are grouped. It may be a state point or a sequence of state points to group by specific state point keys. It may also be an arbitrary callable of :class:`~signac.contrib.job.Job` when greater flexibility is needed. - :type key: - str, Iterable, or callable - :param default: + default : str, Iterable, or callable Default value used for grouping if the key is missing or invalid. - :type default: - str, Iterable, or callable - :param sort_by: + sort_by : str or None State point parameter used to sort the jobs before grouping. The default value is None, which does no sorting. - :type sort_by: - str or None - :param sort_ascending: + sort_ascending : bool Whether jobs are to be sorted in ascending order. The default value is True. - :type sort_ascending: - bool - :param select: + select : callable or None Condition for filtering individual jobs. This is passed as the callable argument to :func:`filter`. The default behavior is no filtering. - :type select: - callable or None + + Returns + ------- + aggregator : :class:`~.aggregator` + The :meth:`~.groupby` aggregator. + """ if isinstance(key, str): if default is None: @@ -265,6 +269,18 @@ def _get_unique_function_id(self, func): It is possible for equivalent functions to have different ids if the bytecode is not identical. + + Parameters + ---------- + func : callable + The function to be hashed. + + Returns + ------- + str : + The hash of the function's bytecode if possible, otherwise the hash + of the function. + """ try: return hash(func.__code__.co_code) @@ -280,10 +296,16 @@ def _create_AggregatesStore(self, project): the classes that actually hold sequences of jobs to which aggregate operations will be applied. - :param project: + Parameters + ---------- + project : class:`flow.FlowProject` or :class:`signac.contrib.project.Project` A signac project used to fetch jobs for creating aggregates. - :type project: - :class:`flow.FlowProject` or :class:`signac.contrib.project.Project` + + Returns + ------- + :class:`~._DefaultAggregateStore` or :class:`~._AggregatesStore` : + The aggregate store. + """ if not self._is_aggregate: return _DefaultAggregateStore(project) @@ -295,10 +317,11 @@ def __call__(self, func=None): This call operator allows the class to be used as a decorator. - :param func: + Parameters + ---------- + func : callable The function to decorate. - :type func: - callable + """ if callable(func): setattr(func, "_flow_aggregate", self) @@ -316,14 +339,13 @@ class _AggregatesStore(Mapping): This is a callable class which, when called, generates all the aggregates. Iterating over this object yields all aggregates. - :param aggregator: + Parameters + ---------- + aggregator : class:`aggregator` aggregator object associated with this class. - :type aggregator: - :class:`aggregator` - :param project: + project : class:`flow.FlowProject` or :class:`signac.contrib.project.Project` A signac project used to fetch jobs for creating aggregates. - :type project: - :class:`flow.FlowProject` or :class:`signac.contrib.project.Project` + """ def __init__(self, aggregator, project): @@ -436,10 +458,11 @@ class _DefaultAggregateStore(Mapping): Iterating over this object yields tuples each containing one job from the project. - :param project: + Parameters + ---------- + project : class:`flow.FlowProject` or :class:`signac.contrib.project.Project` A signac project used to fetch jobs for creating aggregates. - :type project: - :class:`flow.FlowProject` or :class:`signac.contrib.project.Project` + """ def __init__(self, project): @@ -450,7 +473,14 @@ def __iter__(self): yield job.get_id() def __getitem__(self, id): - """Return an aggregate of one job from its job id.""" + """Return an aggregate of one job from its job id. + + Parameters + ---------- + id : str + The job id. + + """ try: return (self._project.open_job(id=id),) except KeyError: @@ -459,10 +489,11 @@ def __getitem__(self, id): def __contains__(self, id): """Return whether this instance contains a job (by job id). - :param id: + Parameters + ---------- + id : str The job id. - :type id: - str + """ try: self._project.open_job(id=id) @@ -506,10 +537,16 @@ def _register_aggregates(self, project): def get_aggregate_id(jobs): """Generate hashed id for an aggregate of jobs. - :param jobs: - The signac job handles - :type jobs: - tuple + Parameters + ---------- + jobs : tuple of :class:`~signac.contrib.job.Job` + The signac job handles. + + Returns + ------- + str : + The generated aggregate id. + """ if len(jobs) == 1: return jobs[0].get_id() # Return job id as it's already unique From 6f4883dfa2adf6244e8829505c765ba3ae2fb6a9 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 7 Dec 2020 22:36:51 -0600 Subject: [PATCH 03/34] Fix name of aggregator class. --- flow/aggregates.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/flow/aggregates.py b/flow/aggregates.py index 8c0508cfe..ef43f5bee 100644 --- a/flow/aggregates.py +++ b/flow/aggregates.py @@ -290,11 +290,11 @@ def _get_unique_function_id(self, func): def _create_AggregatesStore(self, project): """Create the actual collections of jobs to be sent to aggregate operations. - The :class:`aggregate` class is just a decorator that provides a signal for - operation functions that should be treated as aggregate operations and - information on how to perform the aggregation. This function generates - the classes that actually hold sequences of jobs to which aggregate - operations will be applied. + The :class:`aggregator` class is just a decorator that provides a + signal for operation functions that should be treated as aggregate + operations and information on how to perform the aggregation. This + function generates the classes that actually hold sequences of jobs to + which aggregate operations will be applied. Parameters ---------- From 852c1a8fab7671fdc85cd7acadb808f0a08f0818 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 8 Dec 2020 00:15:51 -0600 Subject: [PATCH 04/34] Revisions to environment docstrings. --- flow/environment.py | 195 ++++++++++++++++++++++++++++++++------------ 1 file changed, 142 insertions(+), 53 deletions(-) diff --git a/flow/environment.py b/flow/environment.py index 0012800e7..7ac797285 100644 --- a/flow/environment.py +++ b/flow/environment.py @@ -44,7 +44,7 @@ def setup(py_modules, **attrs): """Set up user-defined environment modules. - Use this function in place of setuptools.setup to not only install + Use this function in place of :meth:`setuptools.setup` to not only install an environment's module, but also register it with the global signac configuration. Once registered, the environment is automatically imported when the :meth:`~flow.get_environment` function is called. @@ -94,7 +94,24 @@ def __init__(cls, name, bases, dct): def template_filter(func): - """Mark the function as a ComputeEnvironment template filter.""" + """Decorate a function as a :class:`~.ComputeEnvironment` template filter. + + This decorator is applied to methods defined in a subclass of + :class:`~.ComputeEnvironment` that are used in that environment's + templates. The decorated function becomes a class method that is available + as a :ref:`jinja2 filter ` in templates rendered by a + :class:`~.FlowProject` with that :class:`~.ComputeEnvironment`. + + Parameters + ---------- + func : callable + Function to decorate. + + Returns + ------- + callable : Decorated function. + + """ setattr(func, "_flow_template_filter", True) return classmethod(func) @@ -148,10 +165,27 @@ def get_scheduler(cls): @classmethod def submit(cls, script, flags=None, *args, **kwargs): - """Submit a job submission script to the environment's scheduler. + r"""Submit a job submission script to the environment's scheduler. Scripts should be submitted to the environment, instead of directly to the scheduler to allow for environment specific post-processing. + + Parameters + ---------- + script : str + The script to submit. + flags : list + A list of additional flags to provide to the scheduler. + (Default value = None) + \*args + Positional arguments forwarded to the scheduler's submit method. + \*\*kwargs + Keyword arguments forwarded to the scheduler's submit method. + + Returns + ------- + JobStatus.submitted or None : Status of job, if submitted. + """ if flags is None: flags = [] @@ -171,10 +205,11 @@ def submit(cls, script, flags=None, *args, **kwargs): def add_args(cls, parser): """Add arguments related to this compute environment to an argument parser. - :param parser: - The argument parser to add arguments to. - :type parser: - :class:`argparse.ArgumentParser` + Parameters + ---------- + parser : :class:`argparse.ArgumentParser` + The argument parser where arguments will be added. + """ return @@ -193,14 +228,25 @@ def get_config_value(cls, key, default=flow_config._GET_CONFIG_VALUE_NONE): be specific to this environment definition. For example, a key should be ``'account'``, not ``'MyEnvironment.account'``. - :param key: The environment specific configuration key. - :type key: str - :param default: A default value in case the key cannot be found + Parameters + ---------- + key : str + The environment specific configuration key. + default : str + A default value in case the key cannot be found within the user's configuration. - :type default: str - :return: The value or default value. - :raises SubmitError: If the key is not in the user's configuration + + Returns + ------- + type + The value or default value. + + Raises + ------ + SubmitError + If the key is not in the user's configuration and no default value is provided. + """ return flow_config.require_config_value(key, ns=cls.__name__, default=default) @@ -208,12 +254,15 @@ def get_config_value(cls, key, default=flow_config._GET_CONFIG_VALUE_NONE): def _get_omp_prefix(cls, operation): """Get the OpenMP prefix based on the `omp_num_threads` directive. - :param operation: - The operation for which to add prefix. - :return omp_prefix: - The prefix should be added for the operation. - :type omp_prefix: - str + Parameters + ---------- + operation : :class:`flow.project._JobOperation` + The operation to be prefixed. + + Returns + ------- + str : The prefix to be added to the operation's command. + """ return "export OMP_NUM_THREADS={}; ".format( operation.directives["omp_num_threads"] @@ -223,20 +272,17 @@ def _get_omp_prefix(cls, operation): def _get_mpi_prefix(cls, operation, parallel): """Get the mpi prefix based on proper directives. - :param operation: - The operation for which to add prefix. - :type operation: - :class:`flow._JobOperation` - :param parallel: - If True, operations are assumed to be executed in parallel, which means - that the number of total tasks is the sum of all tasks instead of the - maximum number of tasks. Default is set to False. - :type parallel: - bool - :return mpi_prefix: - The prefix should be added for the operation. - :type mpi_prefix: - str + Parameters + ---------- + operation : :class:`flow.project._JobOperation` + The operation to be prefixed. + parallel : bool + Unused parameter. TODO: remove this. + + Returns + ------- + str : The prefix to be added to the operation's command. + """ if operation.directives.get("nranks"): return "{} -n {} ".format(cls.mpi_cmd, operation.directives["nranks"]) @@ -246,22 +292,25 @@ def _get_mpi_prefix(cls, operation, parallel): def get_prefix(cls, operation, parallel=False, mpi_prefix=None, cmd_prefix=None): """Template filter for getting the prefix based on proper directives. - :param operation: - The operation for which to add prefix. - :param parallel: + Parameters + ---------- + operation : :class:`flow.project._JobOperation` + The operation to be prefixed. + parallel : bool If True, operations are assumed to be executed in parallel, which means that the number of total tasks is the sum of all tasks instead of the maximum number of tasks. Default is set to False. - :param mpi_prefix: + mpi_prefix : str User defined mpi_prefix string. Default is set to None. This will be deprecated and removed in the future. - :param cmd_prefix: + cmd_prefix : str User defined cmd_prefix string. Default is set to None. This will be deprecated and removed in the future. - :return prefix: - The prefix should be added for the operation. - :type prefix: - str + + Returns + ------- + str : The prefix to be added to the operation's command. + """ prefix = "" if operation.directives.get("omp_num_threads"): @@ -346,7 +395,8 @@ class LSFEnvironment(ComputeEnvironment): class NodesEnvironment(ComputeEnvironment): """A compute environment consisting of multiple compute nodes. - Each compute node is assumed to have a specific number of compute units, e.g., CPUs. + Each compute node is assumed to have a specific number of compute units, + e.g., CPUs. """ @@ -355,7 +405,14 @@ class DefaultTorqueEnvironment(NodesEnvironment, TorqueEnvironment): @classmethod def add_args(cls, parser): - """Add arguments to the parser.""" + """Add arguments to the parser. + + Parameters + ---------- + parser : :class:`argparse.ArgumentParser` + The argument parser where arguments will be added. + + """ super().add_args(parser) parser.add_argument( "-w", "--walltime", type=float, help="The wallclock time in hours." @@ -381,7 +438,14 @@ class DefaultSlurmEnvironment(NodesEnvironment, SlurmEnvironment): @classmethod def add_args(cls, parser): - """Add arguments to the parser.""" + """Add arguments to the parser. + + Parameters + ---------- + parser : :class:`argparse.ArgumentParser` + The argument parser where arguments will be added. + + """ super().add_args(parser) parser.add_argument( "--memory", @@ -413,7 +477,14 @@ class DefaultLSFEnvironment(NodesEnvironment, LSFEnvironment): @classmethod def add_args(cls, parser): - """Add arguments to the parser.""" + """Add arguments to the parser. + + Parameters + ---------- + parser : :class:`argparse.ArgumentParser` + The argument parser where arguments will be added. + + """ super().add_args(parser) parser.add_argument( "-w", @@ -460,7 +531,19 @@ def _import_configured_environments(): def registered_environments(import_configured=True): - """Return a list of registered environments.""" + """Return a list of registered environments. + + Parameters + ---------- + import_configured : bool + Whether to import environments specified in the flow configuration. + (Default value = True) + + Returns + ------- + list : List of registered environments. + + """ if import_configured: _import_configured_environments() return list(ComputeEnvironment.registry.values()) @@ -474,12 +557,18 @@ def get_environment(test=False, import_configured=True): environment where the :meth:`~.ComputeEnvironment.is_present` method returns True. - :param test: - Whether to return the TestEnvironment. - :type test: - bool - :returns: - The detected environment class. + Parameters + ---------- + test : bool + Whether to return the TestEnvironment. (Default value = False) + import_configured : bool + Whether to import environments specified in the flow configuration. + (Default value = True) + + Returns + ------- + :class:`~.ComputeEnvironment` : The detected environment class. + """ if test: return TestEnvironment From 331c2679d32148d0a6b4e47a3c7176b43664bd78 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 8 Dec 2020 00:16:07 -0600 Subject: [PATCH 05/34] Revisions to aggregate docstrings. --- flow/aggregates.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flow/aggregates.py b/flow/aggregates.py index ef43f5bee..ed9b97cc2 100644 --- a/flow/aggregates.py +++ b/flow/aggregates.py @@ -298,7 +298,7 @@ def _create_AggregatesStore(self, project): Parameters ---------- - project : class:`flow.FlowProject` or :class:`signac.contrib.project.Project` + project : :class:`flow.FlowProject` or :class:`signac.contrib.project.Project` A signac project used to fetch jobs for creating aggregates. Returns @@ -341,9 +341,9 @@ class _AggregatesStore(Mapping): Parameters ---------- - aggregator : class:`aggregator` + aggregator : :class:`aggregator` aggregator object associated with this class. - project : class:`flow.FlowProject` or :class:`signac.contrib.project.Project` + project : :class:`flow.FlowProject` or :class:`signac.contrib.project.Project` A signac project used to fetch jobs for creating aggregates. """ @@ -460,7 +460,7 @@ class _DefaultAggregateStore(Mapping): Parameters ---------- - project : class:`flow.FlowProject` or :class:`signac.contrib.project.Project` + project : :class:`flow.FlowProject` or :class:`signac.contrib.project.Project` A signac project used to fetch jobs for creating aggregates. """ From e02370404f9f4fe5f69fa712c822e4112426f43f Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 8 Dec 2020 00:29:47 -0600 Subject: [PATCH 06/34] NumPy-doc-ify INCITE environments. --- flow/environment.py | 6 +-- flow/environments/incite.py | 81 ++++++++++++++++++++++++++----------- 2 files changed, 60 insertions(+), 27 deletions(-) diff --git a/flow/environment.py b/flow/environment.py index 7ac797285..e5b34a79d 100644 --- a/flow/environment.py +++ b/flow/environment.py @@ -252,7 +252,7 @@ def get_config_value(cls, key, default=flow_config._GET_CONFIG_VALUE_NONE): @classmethod def _get_omp_prefix(cls, operation): - """Get the OpenMP prefix based on the `omp_num_threads` directive. + """Get the OpenMP prefix based on the ``omp_num_threads`` directive. Parameters ---------- @@ -270,7 +270,7 @@ def _get_omp_prefix(cls, operation): @classmethod def _get_mpi_prefix(cls, operation, parallel): - """Get the mpi prefix based on proper directives. + """Get the MPI prefix based on the ``nranks`` directives. Parameters ---------- @@ -290,7 +290,7 @@ def _get_mpi_prefix(cls, operation, parallel): @template_filter def get_prefix(cls, operation, parallel=False, mpi_prefix=None, cmd_prefix=None): - """Template filter for getting the prefix based on proper directives. + """Template filter generating a command prefix from directives. Parameters ---------- diff --git a/flow/environments/incite.py b/flow/environments/incite.py index 781197e34..08f08d79c 100644 --- a/flow/environments/incite.py +++ b/flow/environments/incite.py @@ -37,15 +37,21 @@ def my_operation(job): def calc_num_nodes(cls, resource_sets, parallel=False): """Compute the number of nodes needed. - :param resource_sets: + Parameters + ---------- + resource_sets : iterable of tuples Resource sets for each operation, as a sequence of tuples of - (number of sets, number of tasks, CPUs per task, GPUs). - :type resource_sets: - iterable - :param parallel: - Whether operations should run in parallel or serial. - :type parallel: - bool + *(Number of resource sets, tasks (MPI Ranks) per resource set, + physical cores (CPUs) per resource set, GPUs per resource set)*. + parallel : bool + Whether operations should run in parallel or serial. (Default value + = False) + + Returns + ------- + int + Number of nodes needed. + """ nodes_used_final = 0 cores_used = gpus_used = nodes_used = 0 @@ -76,11 +82,23 @@ def calc_num_nodes(cls, resource_sets, parallel=False): def guess_resource_sets(cls, operation): """Determine the resources sets needed for an operation. - :param operation: + Parameters + ---------- + operation : class:`flow.BaseFlowOperation` The operation whose directives will be used to compute the resource set. - :type operation: - :class:`flow.BaseFlowOperation` + + Returns + ------- + int + Number of resource sets. + int + Number of tasks (MPI ranks) per resource set. + int + Number of physical cores (CPUs) per resource set. + int + Number of GPUs per resource set. + """ ntasks = max(operation.directives.get("nranks", 1), 1) np = operation.directives.get("np", ntasks) @@ -107,7 +125,21 @@ def guess_resource_sets(cls, operation): @classmethod def jsrun_options(cls, resource_set): - """Return jsrun options for the provided resource set.""" + """Return jsrun options for the provided resource set. + + Parameters + ---------- + resource_set : + Tuple of *(Number of resource sets, tasks (MPI Ranks) per resource + set, physical cores (CPUs) per resource set, GPUs per resource + set)*. + + Returns + ------- + str + Resource set options. + + """ nsets, tasks, cpus, gpus = resource_set cuda_aware_mpi = ( "--smpiargs='-gpu'" if (nsets > 0 or tasks > 0) and gpus > 0 else "" @@ -116,18 +148,19 @@ def jsrun_options(cls, resource_set): @classmethod def _get_mpi_prefix(cls, operation, parallel): - """Get the mpi prefix based on proper directives. - - :param operation: - The operation for which to add prefix. - :param parallel: - If True, operations are assumed to be executed in parallel, which means - that the number of total tasks is the sum of all tasks instead of the - maximum number of tasks. Default is set to False. - :return mpi_prefix: - The prefix should be added for the operation. - :type mpi_prefix: - str + """Get the jsrun options based on directives. + + Parameters + ---------- + operation : :class:`flow.project._JobOperation` + The operation to be prefixed. + parallel : bool + Unused parameter. TODO: remove this. + + Returns + ------- + str : The prefix to be added to the operation's command. + """ extra_args = str(operation.directives.get("extra_jsrun_args", "")) resource_set = cls.guess_resource_sets(operation) From efcd9eeab964d2321733bd0961a3e46fb8bf925f Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 8 Dec 2020 00:38:19 -0600 Subject: [PATCH 07/34] NumPy-doc-ify remaining environments. --- flow/environment.py | 5 ++- flow/environments/incite.py | 5 ++- flow/environments/umich.py | 9 +++- flow/environments/xsede.py | 85 ++++++++++++++++++++++++++++--------- 4 files changed, 81 insertions(+), 23 deletions(-) diff --git a/flow/environment.py b/flow/environment.py index e5b34a79d..6ce2dbe08 100644 --- a/flow/environment.py +++ b/flow/environment.py @@ -277,7 +277,10 @@ def _get_mpi_prefix(cls, operation, parallel): operation : :class:`flow.project._JobOperation` The operation to be prefixed. parallel : bool - Unused parameter. TODO: remove this. + If True, operations are assumed to be executed in parallel, which + means that the number of total tasks is the sum of all tasks + instead of the maximum number of tasks. Default is set to False. + :return mpi_prefix: The prefix should be added for the operation. Returns ------- diff --git a/flow/environments/incite.py b/flow/environments/incite.py index 08f08d79c..f19549089 100644 --- a/flow/environments/incite.py +++ b/flow/environments/incite.py @@ -155,7 +155,10 @@ def _get_mpi_prefix(cls, operation, parallel): operation : :class:`flow.project._JobOperation` The operation to be prefixed. parallel : bool - Unused parameter. TODO: remove this. + If True, operations are assumed to be executed in parallel, which + means that the number of total tasks is the sum of all tasks + instead of the maximum number of tasks. Default is set to False. + :return mpi_prefix: The prefix should be added for the operation. Returns ------- diff --git a/flow/environments/umich.py b/flow/environments/umich.py index 405d6f243..d570f20b2 100644 --- a/flow/environments/umich.py +++ b/flow/environments/umich.py @@ -17,7 +17,14 @@ class GreatLakesEnvironment(DefaultSlurmEnvironment): @classmethod def add_args(cls, parser): - """Add arguments to parser.""" + """Add arguments to parser. + + Parameters + ---------- + parser : :class:`argparse.ArgumentParser` + The argument parser where arguments will be added. + + """ super().add_args(parser) parser.add_argument( "--partition", diff --git a/flow/environments/xsede.py b/flow/environments/xsede.py index 958365ba8..697883b8f 100644 --- a/flow/environments/xsede.py +++ b/flow/environments/xsede.py @@ -23,7 +23,14 @@ class CometEnvironment(DefaultSlurmEnvironment): @classmethod def add_args(cls, parser): - """Add arguments to parser.""" + """Add arguments to parser. + + Parameters + ---------- + parser : :class:`argparse.ArgumentParser` + The argument parser where arguments will be added. + + """ super().add_args(parser) parser.add_argument( @@ -67,24 +74,51 @@ def return_and_increment(cls, increment): environment variable will be used upon script generation. At run time, the base offset will be set only once (when run initializes the environment). + + Parameters + ---------- + increment : int + The increment to apply to the base offset. + + Returns + ------- + int : The value of the base offset before incrementing. + """ cls.base_offset += increment return cls.base_offset - increment @template_filter - def decrement_offset(cls, value): + def decrement_offset(cls, decrement): """Decrement the offset value. - This function is a hackish solution to get around the fact that Jinja has - very limited support for direct modification of Python objects, and we need - to be able to reset the offset between bundles. + This function is a hackish solution to get around the fact that Jinja + has very limited support for direct modification of Python objects, and + we need to be able to reset the offset between bundles. + + Parameters + ---------- + decrement : int + The decrement to apply to the base offset. + + Returns + ------- + str : Empty string (to render nothing in the jinja template). + """ - cls.base_offset -= int(value) + cls.base_offset -= int(decrement) return "" @classmethod def add_args(cls, parser): - """Add arguments to parser.""" + """Add arguments to parser. + + Parameters + ---------- + parser : :class:`argparse.ArgumentParser` + The argument parser where arguments will be added. + + """ super().add_args(parser) parser.add_argument( "--partition", @@ -112,18 +146,22 @@ def add_args(cls, parser): @classmethod def _get_mpi_prefix(cls, operation, parallel): - """Get the mpi prefix based on proper directives. - - :param operation: - The operation for which to add prefix. - :param parallel: - If True, operations are assumed to be executed in parallel, which means - that the number of total tasks is the sum of all tasks instead of the - maximum number of tasks. Default is set to False. - :return mpi_prefix: - The prefix should be added for the operation. - :type mpi_prefix: - str + """Get the MPI prefix based on directives and an offset. + + Parameters + ---------- + operation : :class:`flow.project._JobOperation` + The operation to be prefixed. + parallel : bool + If True, operations are assumed to be executed in parallel, which + means that the number of total tasks is the sum of all tasks + instead of the maximum number of tasks. Default is set to False. + :return mpi_prefix: The prefix should be added for the operation. + + Returns + ------- + str : The prefix to be added to the operation's command. + """ if operation.directives.get("nranks"): prefix = "{} -n {} -o {} task_affinity ".format( @@ -151,7 +189,14 @@ class BridgesEnvironment(DefaultSlurmEnvironment): @classmethod def add_args(cls, parser): - """Add arguments to parser.""" + """Add arguments to parser. + + Parameters + ---------- + parser : :class:`argparse.ArgumentParser` + The argument parser where arguments will be added. + + """ super().add_args(parser) parser.add_argument( "--partition", From 422abec41fa60e893150410247f2a82f25476778 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 8 Dec 2020 00:54:08 -0600 Subject: [PATCH 08/34] NumPy-doc-ify render_status. --- flow/render_status.py | 153 ++++++++++++++++++++++-------------------- 1 file changed, 80 insertions(+), 73 deletions(-) diff --git a/flow/render_status.py b/flow/render_status.py index 3543308b0..78bfd9308 100644 --- a/flow/render_status.py +++ b/flow/render_status.py @@ -9,10 +9,10 @@ class Renderer: - """A class for rendering status in different format. + """A class for rendering status in a specified style and format. - This class provides method and string output for rendering status output in different format, - currently supports: terminal, markdown and html + This class provides the :meth:`.render` method for rendering status output + in different formats, and currently supports terminal, Markdown and HTML. """ def __init__(self): @@ -23,21 +23,23 @@ def __init__(self): def generate_terminal_output(self): """Get status string in format for terminal. - :return: + Returns + ------- + str Status string in format for terminal. - :rtype: - str + """ self.terminal_output = mistune.terminal(self.markdown_output) return self.terminal_output def generate_html_output(self): - """Get status string in html format. + """Get status string in HTML format. + + Returns + ------- + str + Status string in HTML format. - :return: - Status string in html format. - :rtype: - str """ self.html_output = mistune.html(self.markdown_output) return self.html_output @@ -53,41 +55,34 @@ def render( compact, output_format, ): - """Render the status in different format for print_status. + """Render the status. - :param template: + Parameters + ---------- + template : str User provided Jinja2 template file. - :type template: - str - :param template_environment: + template_environment : :class:`jinja2.Environment` Template environment. - :type template_environment: - :class:`jinja2.Environment` - :param context: - Context that includes all the information for rendering status output. - :type context: - dict - :param detailed: + context : dict + Context that includes all the information for rendering status + output. + detailed : bool Print a detailed status of each job. - :type detailed: - bool - :param expand: + expand : bool Present labels and operations in two separate tables. - :type expand: - bool - :param unroll: + unroll : bool Separate columns for jobs and the corresponding operations. - :type unroll: - bool - :param compact: + compact : bool Print a compact version of the output. - :type compact: - bool - :param output_format: + output_format : str Rendering output format, supports: - 'terminal' (default), 'markdown' or 'html'. - :type output_format: - str + ``'terminal'`` (default), ``'markdown'``, or ``'html'``. + + Returns + ------- + str + Status output. + """ # Use Jinja2 template for status output if template is None: @@ -103,18 +98,22 @@ def render( def draw_progressbar(value, total, escape="", width=40): """Visualize progress with a progress bar. - :param value: + Parameters + ---------- + value : int The current progress as a fraction of total. - :type value: - int - :param total: + total : int The maximum value that 'value' may obtain. - :type total: - int - :param width: - The character width of the drawn progress bar. - :type width: - int + width : int + The character width of the drawn progress bar. (Default value = 40) + escape : str + Escape character needed for some formats. (Default value = "") + + Returns + ------- + str + Formatted progress bar. + """ assert value >= 0 and total > 0 bar_format = escape + f"|{{bar:{width}}}" + escape + "| {percentage:<0.2f}%" @@ -125,18 +124,20 @@ def draw_progressbar(value, total, escape="", width=40): def job_filter(job_op, scheduler_status_code, all_ops): """Filter eligible jobs for status print. - :param job_op: + Parameters + ---------- + job_op : dict Operation information for a job. - :type job_op: - dict - :param scheduler_status_code: + scheduler_status_code : dict Dictionary information for status code. - :type scheduler_status_code: - dict - :param all_ops: + all_ops : bool Boolean value indicate if all operations should be displayed. - :type all_ops: - bool + + Returns + ------- + bool + Whether the job is eligible to print. + """ return ( scheduler_status_code[job_op["scheduler_status"]] != "U" @@ -147,14 +148,18 @@ def job_filter(job_op, scheduler_status_code, all_ops): def get_operation_status(operation_info, symbols): """Determine the status of an operation. - :param operation_info: + Parameters + ---------- + operation_info : dict Dictionary containing operation information. - :type operation_info: - dict - :param symbols: - Dictionary containing code for different job status. - :type symbols: - dict + symbols : dict + Dictionary containing code for different job statuses. + + Returns + ------- + str + The symbol for the job status. + """ if operation_info["scheduler_status"] >= JobStatus.active: op_status = "running" @@ -172,18 +177,20 @@ def get_operation_status(operation_info, symbols): def highlight(string, eligible, pretty): """Change font to bold within jinja2 template. - :param string: + Parameters + ---------- + string : str The string to be printed. - :type string: - str - :param eligible: + eligible : bool Boolean value for job eligibility. - :type eligible: - bool - :param pretty: + pretty : bool Prettify the output. - :type pretty: - bool + + Returns + ------- + str + The highlighted (bold font) string. + """ if eligible and pretty: return "**" + string + "**" From ae16bfcf5bf84a6b0e27298b7fd7a3a6d6ae711a Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 8 Dec 2020 01:01:15 -0600 Subject: [PATCH 09/34] NumPy-doc-ify scheduler base classes. --- flow/scheduling/base.py | 7 +++++-- flow/scheduling/fakescheduler.py | 11 ++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/flow/scheduling/base.py b/flow/scheduling/base.py index bb2411783..e77ed83c3 100644 --- a/flow/scheduling/base.py +++ b/flow/scheduling/base.py @@ -98,7 +98,10 @@ def _prevent_dos(cls): def jobs(self): """Yield all cluster jobs. - :yields: - :class:`.ClusterJob` + Yields + ------ + :class:`.ClusterJob` + Cluster job. + """ raise NotImplementedError() diff --git a/flow/scheduling/fakescheduler.py b/flow/scheduling/fakescheduler.py index 92d7b4542..877f6088f 100644 --- a/flow/scheduling/fakescheduler.py +++ b/flow/scheduling/fakescheduler.py @@ -25,7 +25,16 @@ def jobs(self): return None def submit(self, script, **kwargs): - """Print the script to screen.""" + r"""Print the script to screen. + + Parameters + ---------- + script : str + Script to print. + \*\*kwargs : + Keyword arguments (ignored). + + """ print(script) @classmethod From c0a9f64ea9b15a86ae16be74668326929510df15 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 8 Dec 2020 01:10:41 -0600 Subject: [PATCH 10/34] NumPy-doc-ify LSF scheduler. --- flow/scheduling/lsf.py | 72 +++++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/flow/scheduling/lsf.py b/flow/scheduling/lsf.py index f462209b3..f4aaf4351 100644 --- a/flow/scheduling/lsf.py +++ b/flow/scheduling/lsf.py @@ -32,7 +32,20 @@ def _parse_status(s): def _fetch(user=None): - """Fetch the cluster job status information from the LSF scheduler.""" + """Fetch the cluster job status information from the LSF scheduler. + + Parameters + ---------- + user : str + Limit the status information to cluster jobs submitted by user. + (Default value = None) + + Yields + ------ + :class:`~.LSFJob` + LSF cluster job. + + """ if user is None: user = getpass.getuser() @@ -67,15 +80,18 @@ def name(self): class LSFScheduler(Scheduler): - """Implementation of the abstract Scheduler class for LSF schedulers. + r"""Implementation of the abstract Scheduler class for LSF schedulers. This class can submit cluster jobs to a LSF scheduler and query their current status. - :param user: + Parameters + ---------- + user : str Limit the status information to cluster jobs submitted by user. - :type user: - str + \*\*kwargs + Forwarded to the parent constructor. + """ # The standard command used to submit jobs to the LSF scheduler. @@ -93,28 +109,34 @@ def jobs(self): def submit( self, script, after=None, hold=False, pretend=False, flags=None, **kwargs ): - """Submit a job script for execution to the scheduler. + r"""Submit a job script for execution to the scheduler. - :param script: + Parameters + ---------- + script : str The job script submitted for execution. - :type script: - str - :param after: - Execute the submitted script after a job with this id has completed. - :type after: - str - :param pretend: - If True, do not actually submit the script, but only simulate the submission. - Can be used to test whether the submission would be successful. - Please note: A successful "pretend" submission is not guaranteed to succeed. - :type pretend: - bool - :param flags: - Additional arguments to pass through to the scheduler submission command. - :type flags: - list - :returns: - Returns True if the cluster job was successfully submitted, otherwise None. + after : str + Execute the submitted script after a job with this id has + completed. (Default value = None) + hold : bool + Whether to hold the job upon submission. (Default value = False) + pretend : bool + If True, do not actually submit the script, but only simulate the + submission. Can be used to test whether the submission would be + successful. Please note: A successful "pretend" submission is not + guaranteed to succeed. (Default value = False) + flags : list + Additional arguments to pass through to the scheduler submission + command. (Default value = None) + \*\*kwargs : + Additional keyword arguments (ignored). + + Returns + ------- + bool + Returns True if the cluster job was successfully submitted, + otherwise None. + """ if flags is None: flags = [] From d7ed093347126ef7f387ec3f997ab9786642f312 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 8 Dec 2020 01:12:39 -0600 Subject: [PATCH 11/34] NumPy-doc-ify simple scheduler. --- flow/scheduling/simple_scheduler.py | 30 +++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/flow/scheduling/simple_scheduler.py b/flow/scheduling/simple_scheduler.py index 87723153c..cefe81552 100644 --- a/flow/scheduling/simple_scheduler.py +++ b/flow/scheduling/simple_scheduler.py @@ -37,20 +37,26 @@ def jobs(self): yield ClusterJob(doc["job_name"], JobStatus(doc["status"])) def submit(self, script, pretend=False, **kwargs): - """Submit a job script for execution to the scheduler. + r"""Submit a job script for execution to the scheduler. - :param script: + Parameters + ---------- + script : str The job script submitted for execution. - :type script: - str - :param pretend: - If True, do not actually submit the script, but only simulate the submission. - Can be used to test whether the submission would be successful. - A successful "pretend" submission is not guaranteed to succeed. - :type pretend: - bool - :returns: - Returns True if the cluster job was successfully submitted, otherwise None. + pretend : bool + If True, do not actually submit the script, but only simulate the + submission. Can be used to test whether the submission would be + successful. A successful "pretend" submission is not + guaranteed to succeed. (Default value = False) + \*\*kwargs : + Keyword arguments (ignored). + + Returns + ------- + bool + Returns True if the cluster job was successfully submitted, + otherwise None. + """ cmd = self.cmd + ["submit"] if pretend: From 20bcbe0beed1640bc084c09ce4e9daad00a38341 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 8 Dec 2020 01:16:27 -0600 Subject: [PATCH 12/34] NumPy-doc-ify SLURM scheduler. --- flow/scheduling/slurm.py | 72 ++++++++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/flow/scheduling/slurm.py b/flow/scheduling/slurm.py index 55dd4df6c..34aad933a 100644 --- a/flow/scheduling/slurm.py +++ b/flow/scheduling/slurm.py @@ -18,7 +18,20 @@ def _fetch(user=None): - """Fetch the cluster job status information from the SLURM scheduler.""" + """Fetch the cluster job status information from the SLURM scheduler. + + Parameters + ---------- + user : str + Limit the status information to cluster jobs submitted by user. + (Default value = None) + + Yields + ------ + :class:`~.SlurmJob` + SLURM cluster job. + + """ def parse_status(s): s = s.strip() @@ -60,15 +73,18 @@ class SlurmJob(ClusterJob): class SlurmScheduler(Scheduler): - """Implementation of the abstract Scheduler class for SLURM schedulers. + r"""Implementation of the abstract Scheduler class for SLURM schedulers. This class can submit cluster jobs to a SLURM scheduler and query their current status. - :param user: + Parameters + ---------- + user : str Limit the status information to cluster jobs submitted by user. - :type user: - str + \*\*kwargs + Forwarded to the parent constructor. + """ # The standard command used to submit jobs to the SLURM scheduler. @@ -86,28 +102,34 @@ def jobs(self): def submit( self, script, after=None, hold=False, pretend=False, flags=None, **kwargs ): - """Submit a job script for execution to the scheduler. + r"""Submit a job script for execution to the scheduler. - :param script: + Parameters + ---------- + script : str The job script submitted for execution. - :type script: - str - :param after: - Execute the submitted script after a job with this id has completed. - :type after: - str - :param pretend: - If True, do not actually submit the script, but only simulate the submission. - Can be used to test whether the submission would be successful. - Please note: A successful "pretend" submission is not guaranteed to succeed. - :type pretend: - bool - :param flags: - Additional arguments to pass through to the scheduler submission command. - :type flags: - list - :returns: - Returns True if the cluster job was successfully submitted, otherwise None. + after : str + Execute the submitted script after a job with this id has + completed. (Default value = None) + hold : bool + Whether to hold the job upon submission. (Default value = False) + pretend : bool + If True, do not actually submit the script, but only simulate the + submission. Can be used to test whether the submission would be + successful. Please note: A successful "pretend" submission is not + guaranteed to succeed. (Default value = False) + flags : list + Additional arguments to pass through to the scheduler submission + command. (Default value = None) + \*\*kwargs : + Additional keyword arguments (ignored). + + Returns + ------- + bool + Returns True if the cluster job was successfully submitted, + otherwise None. + """ if flags is None: flags = [] From 1478e0b78e11633418cae6557619b7e1ff530c73 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 8 Dec 2020 01:22:09 -0600 Subject: [PATCH 13/34] NumPy-doc-ify Torque scheduler. --- flow/scheduling/fakescheduler.py | 2 +- flow/scheduling/lsf.py | 2 +- flow/scheduling/simple_scheduler.py | 2 +- flow/scheduling/slurm.py | 2 +- flow/scheduling/torque.py | 74 +++++++++++++++++++---------- 5 files changed, 53 insertions(+), 29 deletions(-) diff --git a/flow/scheduling/fakescheduler.py b/flow/scheduling/fakescheduler.py index 877f6088f..524bb9103 100644 --- a/flow/scheduling/fakescheduler.py +++ b/flow/scheduling/fakescheduler.py @@ -31,7 +31,7 @@ def submit(self, script, **kwargs): ---------- script : str Script to print. - \*\*kwargs : + \*\*kwargs Keyword arguments (ignored). """ diff --git a/flow/scheduling/lsf.py b/flow/scheduling/lsf.py index f4aaf4351..54586ce14 100644 --- a/flow/scheduling/lsf.py +++ b/flow/scheduling/lsf.py @@ -128,7 +128,7 @@ def submit( flags : list Additional arguments to pass through to the scheduler submission command. (Default value = None) - \*\*kwargs : + \*\*kwargs Additional keyword arguments (ignored). Returns diff --git a/flow/scheduling/simple_scheduler.py b/flow/scheduling/simple_scheduler.py index cefe81552..30d5e4233 100644 --- a/flow/scheduling/simple_scheduler.py +++ b/flow/scheduling/simple_scheduler.py @@ -48,7 +48,7 @@ def submit(self, script, pretend=False, **kwargs): submission. Can be used to test whether the submission would be successful. A successful "pretend" submission is not guaranteed to succeed. (Default value = False) - \*\*kwargs : + \*\*kwargs Keyword arguments (ignored). Returns diff --git a/flow/scheduling/slurm.py b/flow/scheduling/slurm.py index 34aad933a..b9bf61a01 100644 --- a/flow/scheduling/slurm.py +++ b/flow/scheduling/slurm.py @@ -121,7 +121,7 @@ def submit( flags : list Additional arguments to pass through to the scheduler submission command. (Default value = None) - \*\*kwargs : + \*\*kwargs Additional keyword arguments (ignored). Returns diff --git a/flow/scheduling/torque.py b/flow/scheduling/torque.py index 2d3a0ff79..d32ac9e67 100644 --- a/flow/scheduling/torque.py +++ b/flow/scheduling/torque.py @@ -20,7 +20,20 @@ def _fetch(user=None): - """Fetch the cluster job status information from the TORQUE scheduler.""" + """Fetch the cluster job status information from the TORQUE scheduler. + + Parameters + ---------- + user : str + Limit the status information to cluster jobs submitted by user. + (Default value = None) + + Yields + ------ + :class:`~.TorqueJob` + Torque cluster job. + + """ if user is None: user = getpass.getuser() cmd = f"qstat -fx -u {user}" @@ -77,15 +90,18 @@ def status(self): class TorqueScheduler(Scheduler): - """Implementation of the abstract Scheduler class for TORQUE schedulers. + r"""Implementation of the abstract Scheduler class for TORQUE schedulers. This class can submit cluster jobs to a TORQUE scheduler and query their current status. - :param user: + Parameters + ---------- + user : str Limit the status information to cluster jobs submitted by user. - :type user: - str + \*\*kwargs + Forwarded to the parent constructor. + """ # The standard command used to submit jobs to the TORQUE scheduler. @@ -105,28 +121,36 @@ def jobs(self): def submit( self, script, after=None, pretend=False, hold=False, flags=None, *args, **kwargs ): - """Submit a job script for execution to the scheduler. + r"""Submit a job script for execution to the scheduler. - :param script: + Parameters + ---------- + script : str The job script submitted for execution. - :type script: - str - :param after: - Execute the submitted script after a job with this id has completed. - :type after: - str - :param pretend: - If True, do not actually submit the script, but only simulate the submission. - Can be used to test whether the submission would be successful. - Please note: A successful "pretend" submission is not guaranteed to succeed. - :type pretend: - bool - :param flags: - Additional arguments to pass through to the scheduler submission command. - :type flags: - list - :returns: - The cluster job id if the script was successfully submitted, otherwise None. + after : str + Execute the submitted script after a job with this id has + completed. (Default value = None) + pretend : bool + If True, do not actually submit the script, but only simulate the + submission. Can be used to test whether the submission would be + successful. Please note: A successful "pretend" submission is not + guaranteed to succeed. (Default value = False) + hold : bool + Whether to hold the job upon submission. (Default value = False) + flags : list + Additional arguments to pass through to the scheduler submission + command. (Default value = None) + \*args + Additional positional arguments (ignored). + \*\*kwargs + Additional keyword arguments (ignored). + + Returns + ------- + str + The cluster job id if the script was successfully submitted, + otherwise None. + """ if flags is None: flags = [] From bce58b3406953500083e0da16f2ab40ad66da060 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 8 Dec 2020 11:15:55 -0600 Subject: [PATCH 14/34] NumPy-doc-ify template. --- flow/template.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/flow/template.py b/flow/template.py index 66188edf4..f0cc51cc9 100644 --- a/flow/template.py +++ b/flow/template.py @@ -30,7 +30,29 @@ def init(alias=None, template=None, root=None, out=None): - """Initialize a templated :class:`~.FlowProject` module.""" + """Initialize a templated :class:`~.FlowProject` module. + + Parameters + ---------- + alias : str + Python identifier used as a file name for the template output. Uses + `"project"` if None. (Default value = None) + template : str + Name of the template to use. Uses `"minimal"` if None. (Default value + = None) + root : str + Directory where the output file is placed. Uses the current working + directory if None. (Default value = None) + out : file-like object + The stream where output is printed. Uses ``sys.stderr`` if None. + (Default value = None) + + Returns + ------- + list + List of file names created. + + """ if alias is None: alias = "project" elif not alias.isidentifier(): From e7607a7769333938641800d0b4a140a2c1005a88 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 8 Dec 2020 11:24:38 -0600 Subject: [PATCH 15/34] NumPy-doc-ify testing. --- flow/template.py | 13 +++++++++---- flow/testing.py | 19 ++++++++++++++++++- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/flow/template.py b/flow/template.py index f0cc51cc9..b0333bfbd 100644 --- a/flow/template.py +++ b/flow/template.py @@ -36,15 +36,20 @@ def init(alias=None, template=None, root=None, out=None): ---------- alias : str Python identifier used as a file name for the template output. Uses - `"project"` if None. (Default value = None) + ``"project"`` if None. (Default value = None) template : str - Name of the template to use. Uses `"minimal"` if None. (Default value - = None) + Name of the template to use. Built-in templates are: + + * ``"minimal"`` + * ``"example"`` + * ``"testing"`` + + Uses ``"minimal"`` if None. (Default value = None) root : str Directory where the output file is placed. Uses the current working directory if None. (Default value = None) out : file-like object - The stream where output is printed. Uses ``sys.stderr`` if None. + The stream where output is printed. Uses :obj:`sys.stderr` if None. (Default value = None) Returns diff --git a/flow/testing.py b/flow/testing.py index 900ac815f..b373f760d 100644 --- a/flow/testing.py +++ b/flow/testing.py @@ -8,10 +8,27 @@ def make_project(alias="project", root=None, **kwargs): - """Initialize a project for testing. + r"""Initialize a project for testing. The initialized project has a few operations and a few jobs that are in various points in the workflow defined by the project. + + Parameters + ---------- + alias : str + Python identifier used as a file name for the template output. + (Default value = ``"project"``) + root : str + Directory where the output file is placed. Uses the current working + directory if None. (Default value = None) + \*\*kwargs + Keyword arguments forwarded to :meth:`signac.testing.init_jobs`. + + Returns + ------- + :class:`signac.Project` + Project with initialized jobs. + """ init(alias=alias, root=root, template="testing") project = signac.init_project(name=alias, root=root) From 4a74d17f43febef34bb68d0295fcc65e59f8a69a Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 8 Dec 2020 12:04:32 -0600 Subject: [PATCH 16/34] NumPy-doc-ify util. --- flow/environments/xsede.py | 3 +- flow/util/config.py | 50 ++++++++++---- flow/util/misc.py | 90 ++++++++++++++++-------- flow/util/template_filters.py | 126 +++++++++++++++++++++++----------- 4 files changed, 185 insertions(+), 84 deletions(-) diff --git a/flow/environments/xsede.py b/flow/environments/xsede.py index 697883b8f..56189db75 100644 --- a/flow/environments/xsede.py +++ b/flow/environments/xsede.py @@ -103,7 +103,8 @@ def decrement_offset(cls, decrement): Returns ------- - str : Empty string (to render nothing in the jinja template). + str + Empty string (to render nothing in the jinja template). """ cls.base_offset -= int(decrement) diff --git a/flow/util/config.py b/flow/util/config.py index e21bd7fb7..1e5b00d54 100644 --- a/flow/util/config.py +++ b/flow/util/config.py @@ -26,17 +26,29 @@ class _GetConfigValueNoneType: def require_config_value(key, ns=None, default=_GET_CONFIG_VALUE_NONE): - """Request a value from the user's configuration, fail if not available. + """Request a value from the user's configuration, failing if not available. - :param key: The environment specific configuration key. - :type key: str - :param ns: The namespace in which to look for the key. - :type ns: str - :param default: A default value in case the key cannot be found + Parameters + ---------- + key : str + The environment specific configuration key. + ns : str + The namespace in which to look for the key. (Default value = None) + default : + A default value in case the key cannot be found within the user's configuration. - :return: The value or default value. - :raises ConfigKeyError: If the key is not in the user's configuration + + Returns + ------- + object + The value or default value. + + Raises + ------ + :class:`~.ConfigKeyError` + If the key is not in the user's configuration and no default value is provided. + """ try: if ns is None: @@ -54,12 +66,20 @@ def require_config_value(key, ns=None, default=_GET_CONFIG_VALUE_NONE): def get_config_value(key, ns=None, default=None): """Request a value from the user's configuration. - :param key: The configuration key. - :type key: str - :param ns: The namespace in which to look for the key. - :type ns: str - :param default: A default value in case the key cannot be found - within the user's configuration. - :return: The value if found, None if not found. + Parameters + ---------- + key : str + The configuration key. + ns : str + The namespace in which to look for the key. (Default value = None) + default + A default value returned if the key cannot be found within the user + configuration. + + Returns + ------- + object + The value if found, None if not found. + """ return require_config_value(key=key, ns=ns, default=default) diff --git a/flow/util/misc.py b/flow/util/misc.py index 8beb60547..27e39e1ba 100644 --- a/flow/util/misc.py +++ b/flow/util/misc.py @@ -11,15 +11,25 @@ def _positive_int(value): - """Expect a command line argument to be a positive integer. + """Parse a command line argument as a positive integer. - Designed to be used in conjunction with an argparse.ArgumentParser. + Designed to be used in conjunction with :class:`argparse.ArgumentParser`. + + Parameters + ---------- + value : str + The value to parse. + + Returns + ------- + int + The provided value, cast to an integer. + + Raises + ------ + :class:`argparse.ArgumentTypeError` + If value cannot be cast to an integer or is negative. - :param value: - This function will raise an argparse.ArgumentTypeError if value - is not a positive integer. - :raises: - :class:`argparse.ArgumentTypeError` """ try: ivalue = int(value) @@ -44,21 +54,23 @@ def write_human_readable_statepoint(script, job): def redirect_log(job, filename="run.log", formatter=None, logger=None): """Redirect all messages logged via the logging interface to the given file. - :param job: - An instance of a signac job. - :type job: - :class:`signac.Project.Job` - :formatter: - The logging formatter to use, uses a default formatter if this argument - is not provided. - :type formatter: - :class:`logging.Formatter` - :param logger: - The instance of logger to which the new file log handler is added. Defaults - to the default logger returned by `logging.getLogger()` if this argument is - not provided. - :type logger: - :class:`logging.Logger` + This method is a context manager. The logging handler is removed when + exiting the context. + + Parameters + ---------- + job : :class:`signac.contrib.job.Job` + The signac job whose workspace will store the redirected logs. + filename : str + File name of the log. (Default value = "run.log") + formatter : :class:`logging.Formatter` + The logging formatter to use, uses a default formatter if None. + (Default value = None) + logger : :class:`logging.Logger` + The instance of logger to which the new file log handler is added. + Defaults to the default logger returned by :meth:`logging.getLogger` if + this argument is not provided. + """ if formatter is None: formatter = logging.Formatter( @@ -78,7 +90,17 @@ def redirect_log(job, filename="run.log", formatter=None, logger=None): @contextmanager def add_path_to_environment_pythonpath(path): - """Temporarily insert the current working directory into the environment PYTHONPATH variable.""" + """Insert the provided path into the environment PYTHONPATH variable. + + This method is a context manager. It restores the previous PYTHONPATH when + exiting the context. + + Parameters + ---------- + path : str + Path to add to PYTHONPATH. + + """ path = os.path.realpath(path) pythonpath = os.environ.get("PYTHONPATH") if pythonpath: @@ -112,7 +134,18 @@ def add_cwd_to_environment_pythonpath(): @contextmanager def switch_to_directory(root=None): - """Temporarily switch into the given root directory (if not None).""" + """Temporarily switch into the given root directory (if not None). + + This method is a context manager. It switches to the previous working + directory when exiting the context. + + Parameters + ---------- + root : str + Current working directory to use for within the context. (Default value + = None) + + """ if root is None: yield else: @@ -185,13 +218,14 @@ def _to_hashable(obj): Parameters ---------- - obj - Object to create a hashable version of. Lists are converted - to tuples, and hashes are defined for dicts. + obj : + Object to make hashable. Lists are converted to tuples, and hashes are + defined for dicts. Returns ------- - Hash created for obj. + object + Hashable object. """ if type(obj) is list: diff --git a/flow/util/template_filters.py b/flow/util/template_filters.py index 3bc1c2369..4ab77792e 100644 --- a/flow/util/template_filters.py +++ b/flow/util/template_filters.py @@ -59,23 +59,34 @@ def calc_tasks(operations, name, parallel=False, allow_mixed=False): Calculates the number of tasks for a specific processing unit requested in the operations' directive, e.g., 'np' or 'ngpu'. - :param operations: - The operations for which to calculate the total number of required tasks. - :param name: - The name of the processing unit to calculate the tasks for, e.g., 'np' or 'ngpu'. - :param parallel: + Parameters + ---------- + operations : :class:`~._JobOperation` + The operations used to calculate the total number of required tasks. + name : str + The name of the processing unit to calculate the tasks for, e.g., 'np' + or 'ngpu'. + parallel : bool If True, operations are assumed to be executed in parallel, which means that the number of total tasks is the sum of all tasks instead of the - maximum number of tasks. - :param allow_mixed: + maximum number of tasks. (Default value = False) + allow_mixed : bool By default, the number of requested processing units must be identical for all operations. Unless the argument to this parameter is False, a RuntimeError will be raised if there are mixed requirements. - :returns: + + Returns + ------- + int The number of total tasks required for the specified processing unit. - :raises RuntimeError: - Raises a RuntimeError if the required processing units across operations - is not identical unless the allow_mixed parameter is set to True. + + Raises + ------ + RuntimeError + Raises a RuntimeError if the required processing units across + operations is not identical, unless the ``allow_mixed`` parameter is + set to True. + """ processing_units = [ op.directives[name] * op.directives.get("processor_fraction", 1) @@ -105,21 +116,30 @@ def check_utilization(nn, np, ppn, threshold=0.9, name=None): node utilization is below the given threshold or if the number of calculated required nodes is zero. - :param nn: + Parameters + ---------- + nn : int Number of requested nodes. - :param np: + np : int Number of required processing units (e.g. CPUs, GPUs). - :param ppn: + ppn : int Number of processing units available per node. - :param threshold: - The minimally required node utilization. - :param name: - A human-friendly name for the tested processing unit - to be used in the error message, for example: CPU or GPU. - :returns: + threshold : float + The minimum required node utilization. (Default value = 0.9) + name : str + A human-friendly name for the tested processing unit to be used in the + error message, for example: CPU or GPU. (Default value = None) + + Returns + ------- + int The number of calculated nodes. - :raises RuntimeError: + + Raises + ------ + RuntimeError Raised if the node utilization is below the given threshold. + """ if not (0 <= threshold <= 1.0): raise ValueError("The value for 'threshold' must be between 0 and 1.") @@ -155,21 +175,29 @@ def check_utilization(nn, np, ppn, threshold=0.9, name=None): def calc_num_nodes(np, ppn=1, threshold=0, name=None): """Calculate the number of required nodes with optional utilization check. - :param np: + Parameters + ---------- + np : int Number of required processing units (e.g. CPUs, GPUs). - :param ppn: - Number of processing units available per node. - :param threshold: - (optional) The required node utilization. - The default is 0, which means no check. - :param name: - (optional) A human-friendly name for the tested processing unit - to be used in the error message in case of underutilization. - For example: CPU or GPU. - :returns: + ppn : int + Number of processing units available per node. (Default value = 1) + threshold : float + The required node utilization. The default is 0, which means no check. + name : str + A human-friendly name for the tested processing unit to be + used in the error message in case of underutilization. For example: + CPU or GPU. (Default value = None) + + Returns + ------- + int The number of required nodes. - :raises RuntimeError: + + Raises + ------ + RuntimeError If the calculated node utilization is below the given threshold. + """ nn = int(ceil(np / ppn)) return check_utilization(nn, np, ppn, threshold, name) @@ -178,8 +206,16 @@ def calc_num_nodes(np, ppn=1, threshold=0, name=None): def print_warning(msg): """Print warning message within jinja2 template. - :param: - The warning message as a string + Parameters + ---------- + msg : str + Warning to print. + + Returns + ------- + str + Empty string (to render nothing in the jinja template). + """ import logging @@ -194,14 +230,24 @@ def print_warning(msg): def get_account_name(environment, required=False): """Get account name for environment with user-friendly messages on failure. - :param environment: + Parameters + ---------- + environment : :class:`~.ComputeEnvironment` The environment for which to obtain the account variable. - :param required: + required : bool Specify whether the account name is required instead of optional. - :returns: + (Default value = False) + + Returns + ------- + str The account name for the given environment or None if missing and not required. - :raises SubmitError: - Raised if 'required' is True and the account name is missing. + + Raises + ------ + :class:`~.SubmitError` + Raised if ``required`` is True and the account name is missing. + """ env_name = environment.__name__ try: From e6d3b0f85787f43f39f66bfc5835d2f19b1046ed Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 8 Dec 2020 20:59:34 -0600 Subject: [PATCH 17/34] NumPy-doc-ify project. --- flow/project.py | 1660 ++++++++++++++++++++++++----------------------- 1 file changed, 862 insertions(+), 798 deletions(-) diff --git a/flow/project.py b/flow/project.py index 33ad5d318..9bc080946 100644 --- a/flow/project.py +++ b/flow/project.py @@ -69,7 +69,7 @@ # The TEMPLATE_HELP can be shown with the --template-help option available to all -# command line sub commands that use the templating system. +# command line subcommands that use the templating system. TEMPLATE_HELP = """Execution and submission scripts are generated with the jinja2 template files. Standard files are shipped with the package, but maybe replaced or extended with custom templates provided within a project. @@ -118,10 +118,10 @@ def __invert__(self): """Check all conditions.""" PRE = 1 - """Ignore pre-conditions.""" + """Ignore preconditions.""" POST = 2 - """Ignore post-conditions.""" + """Ignore postconditions.""" ALL = PRE | POST """Ignore all conditions.""" @@ -234,6 +234,20 @@ def _make_bundles(operations, size=None): This utility function splits an iterable of operations into equally sized bundles. The final bundle may be smaller than the specified size. + + Parameters + ---------- + operations : iterable + Iterable of operations. + + size : int + Size of bundles. (Default value = None) + + Yields + ------ + list + Bundles of operations with specified size. + """ if size == 0: size = None @@ -259,29 +273,22 @@ class _JobOperation: This class is used by the :class:`~.FlowGroup` class for the execution and submission process and should not be instantiated by users themselves. - :param id: + Parameters + ---------- + id : str The id of this _JobOperation instance. The id should be unique. - :type id: - str - :param name: + name : str The name of the _JobOperation. - :type name: - str - :param jobs: + jobs : tuple of :class:`~signac.contrib.job.Job` The jobs associated with this operation. - :type jobs: - tuple of :class:`~signac.contrib.job.Job` - :param cmd: + cmd : callable or str The command that executes this operation. Can be a callable that when evaluated returns a string. - :type cmd: - callable or str - :param directives: + directives : :class:`flow.directives._Directives` A :class:`flow.directives._Directives` object of additional parameters that provide instructions on how to execute this operation, e.g., specifically required resources. - :type directives: - :class:`flow.directives._Directives` + """ def __init__(self, id, name, jobs, cmd, directives=None): @@ -381,29 +388,22 @@ class JobOperation(_JobOperation): This class is used by the :class:`~.FlowGroup` class for the execution and submission process and should not be instantiated by users themselves. - :param id: + Parameters + ---------- + id : str The id of this JobOperation instance. The id should be unique. - :type id: - str - :param name: + name : str The name of the JobOperation. - :type name: - str - :param job: + job : :class:`~signac.contrib.job.Job` The job associated with this operation. - :type job: - :class:`~signac.contrib.job.Job` - :param cmd: + cmd : callable or str The command that executes this operation. Can be a callable that when evaluated returns a string. - :type cmd: - callable or str - :param directives: + directives : :class:`flow.directives._Directives` A :class:`flow.directives._Directives` object of additional parameters that provide instructions on how to execute this operation, e.g., specifically required resources. - :type directives: - :class:`flow.directives._Directives` + """ def __init__(self, id, name, job, cmd, directives=None): @@ -456,28 +456,25 @@ class _SubmissionJobOperation(_JobOperation): that will be executed via the "run" command. These groups are known at submission time. - :param \*args: + Parameters + ---------- + \*args Passed to the constructor of :class:`_JobOperation`. - :param eligible_operations: + eligible_operations : list A list of :class:`_JobOperation` that will be executed when this submitted job is executed. - :type eligible_operations: - list - :param operations_with_unmet_preconditions: + operations_with_unmet_preconditions : list A list of :class:`_JobOperation` that will not be executed in the first pass of :meth:`FlowProject.run` due to unmet preconditions. These operations may be executed in subsequent iterations of the run loop. - :type operations_with_unmet_preconditions: - list - :param operations_with_met_postconditions: + operations_with_met_postconditions : list A list of :class:`_JobOperation` that will not be executed in the first pass of :meth:`FlowProject.run` because all postconditions are met. These operations may be executed in subsequent iterations of the run loop. - :type operations_with_met_postconditions: - list - :param \*\*kwargs: + \*\*kwargs : Passed to the constructor of :class:`_JobOperation`. + """ def __init__( @@ -511,14 +508,15 @@ def __init__( class _FlowCondition: """A _FlowCondition represents a condition as a function of a signac job. - The __call__() function of a _FlowCondition object may return either True - or False, representing whether the condition is met or not. - This can be used to build a graph of conditions and operations. + The ``__call__()`` method of a _FlowCondition object may return either True + or False, representing whether the condition is met or not. This can be + used to build a graph of conditions and operations. - :param callback: + Parameters + ---------- + callback : callable A callable with one positional argument (the job). - :type callback: - callable + """ def __init__(self, callback): @@ -544,31 +542,30 @@ def __eq__(self, other): class BaseFlowOperation: - """A BaseFlowOperation represents a data space operation, operating on any job. + """A BaseFlowOperation represents a data space operation acting on any job. Every BaseFlowOperation is associated with a specific command. - Pre-conditions (pre) and post-conditions (post) can be used to - trigger an operation only when certain conditions are met. Conditions are unary - callables, which expect an instance of job as their first and only positional - argument and return either True or False. + Preconditions (pre) and postconditions (post) can be used to trigger an + operation only when certain conditions are met. Conditions are unary + callables, which expect an instance of job as their first and only + positional argument and return either True or False. - An operation is considered "eligible" for execution when all pre-conditions - are met and when at least one of the post-conditions is not met. - Pre-conditions are always met when the list of pre-conditions is empty. - Post-conditions are never met when the list of post-conditions is empty. + An operation is considered "eligible" for execution when all preconditions + are met and when at least one of the postconditions is not met. + Preconditions are always met when the list of preconditions is empty. + Postconditions are never met when the list of postconditions is empty. .. note:: This class should not be instantiated directly. - :param pre: - List of pre-conditions. - :type pre: - sequence of callables - :param post: - List of post-conditions. - :type post: - sequence of callables + Parameters + ---------- + pre : sequence of callables + List of preconditions. + post : sequence of callables + List of postconditions. + """ def __init__(self, pre=None, post=None): @@ -583,18 +580,23 @@ def __init__(self, pre=None, post=None): def _eligible(self, jobs, ignore_conditions=IgnoreConditions.NONE): """Determine eligibility of jobs. - Jobs are eligible when all pre-conditions are true and at least one - post-condition is false, or corresponding conditions are ignored. + Jobs are eligible when all preconditions are true and at least one + postcondition is false, or corresponding conditions are ignored. - :param jobs: + Parameters + ---------- + jobs : tuple The signac job handles. - :type jobs: - tuple - :param ignore_conditions: - Specify if pre and/or post conditions are to be ignored when determining eligibility. - The default is :class:`IgnoreConditions.NONE`. - :type ignore_conditions: - :class:`~.IgnoreConditions` + ignore_conditions : :class:`~.IgnoreConditions` + Specify if preconditions and/or postconditions are to be ignored + when determining eligibility. The default is + :class:`IgnoreConditions.NONE`. + + Returns + ------- + bool + Whether the job is eligible. + """ if not isinstance(ignore_conditions, IgnoreConditions): raise ValueError( @@ -619,31 +621,29 @@ def _eligible(self, jobs, ignore_conditions=IgnoreConditions.NONE): def eligible(self, job, ignore_conditions=IgnoreConditions.NONE): """Determine eligibility of jobs. - Jobs are eligible when all pre-conditions are true and at least one - post-condition is false, or corresponding conditions are ignored. + Jobs are eligible when all preconditions are true and at least one + postcondition is false, or corresponding conditions are ignored. + + Parameters + ---------- + job : :class:`~signac.contrib.job.Job` + The signac job handle. + ignore_conditions : :class:`~.IgnoreConditions` + Specify if pre and/or postconditions check is to be ignored for + eligibility check. The default is :class:`IgnoreConditions.NONE`. - :param job: - The signac job handles. - :type job: - :class:`~signac.contrib.job.Job` - :param ignore_conditions: - Specify if pre and/or post conditions check is to be ignored for - eligibility check. The default is - :class:`IgnoreConditions.NONE`. - :type ignore_conditions: - :class:`~.IgnoreConditions` """ return self._eligible((job,), ignore_conditions) def _complete(self, jobs): - """Check if all post-conditions are met.""" + """Check if all postconditions are met.""" if len(self._postconditions) > 0: return all(cond(jobs) for cond in self._postconditions) return False @deprecated(deprecated_in="0.11", removed_in="0.13", current_version=__version__) def complete(self, job): - """Check if all post-conditions are met.""" + """Check if all postconditions are met.""" return self._complete((job,)) @@ -662,20 +662,17 @@ class FlowCmdOperation(BaseFlowOperation): .. note:: This class should not be instantiated directly. - :param cmd: + Parameters + ---------- + cmd : str or callable The command to execute the operation. Callable values should be a function of ``job``. String values will be formatted with ``cmd.format(job=job)``. - :type cmd: - str or callable - :param pre: - List of pre-conditions. - :type pre: - sequence of callables - :param post: - List of post-conditions. - :type post: - sequence of callables + pre : sequence of callables + List of preconditions. + post : sequence of callables + List of postconditions. + """ def __init__(self, cmd, pre=None, post=None): @@ -714,18 +711,15 @@ class FlowOperation(BaseFlowOperation): .. note:: This class should not be instantiated directly. - :param op_func: + Parameters + ---------- + op_func : callable A callable function of ``*jobs``. - :type op_func: - callable - :param pre: - List of pre-conditions. - :type pre: - sequence of callables - :param post: - List of post-conditions. - :type post: - sequence of callables + pre : sequence of callables + List of preconditions. + post : sequence of callables + List of postconditions. + """ def __init__(self, op_func, pre=None, post=None): @@ -738,10 +732,16 @@ def __str__(self): def __call__(self, *jobs): r"""Call the operation on the provided jobs. - :param \*jobs: + Parameters + ---------- + \*jobs : One or more instances of :class:`.Job`. The jobs passed to the operation. - :type \*jobs: - One or more instances of :class:`.Job`. + + Returns + ------- + object + The result of the operation function. + """ return self._op_func(*jobs) @@ -758,20 +758,17 @@ class FlowGroupEntry: directives. This overrides the default directives specified by :meth:`flow.directives`. - :param name: + Parameters + ---------- + name : str The name of the :class:`FlowGroup` to be created. - :type name: - str - :param options: + options : str The :meth:`FlowProject.run` options to pass when submitting the group. These will be included in all submissions. Submissions use run commands to execute. - :type options: - str - :param aggregator: + aggregator : :class:`~.aggregator` aggregator object associated with the :class:`FlowGroup` - :type aggregator: - :class:`aggregator` + """ def __init__(self, name, options="", aggregator=aggregator.groupsof(1)): @@ -813,16 +810,18 @@ def _set_directives(self, func, directives): def with_directives(self, directives): """Return a decorator that sets group specific directives to the operation. - :param directives: + Parameters + ---------- + directives : dict Directives to use for resource requests and running the operation through the group. - :type directives: - dict - :returns: + + Returns + ------- + function A decorator which registers the function into the group with specified directives. - :rtype: - function + """ def decorator(func): @@ -835,11 +834,13 @@ def decorator(func): class FlowGroup: """A FlowGroup represents a subset of a workflow for a project. - Any :class:`FlowGroup` is associated with one or more instances of + A :class:`FlowGroup` is associated with one or more instances of :class:`BaseFlowOperation`. + Examples + -------- In the example below, the directives will be ``{'nranks': 4}`` for op1 and - ``{'nranks': 2, 'executable': 'python3'}`` for op2 + ``{'nranks': 2, 'executable': 'python3'}`` for op2. .. code-block:: python @@ -857,30 +858,25 @@ def op1(job): def op2(job): pass - :param name: + Parameters + ---------- + name : str The name of the group to be used when calling from the command line. - :type name: - str - :param operations: + operations : dict A dictionary of operations where the keys are operation names and each value is a :class:`BaseFlowOperation`. - :type operations: - dict - :param operation_directives: + operation_directives : dict A dictionary of additional parameters that provide instructions on how to execute a particular operation, e.g., specifically required - resources. Operation names are keys and the dictionaries of directives are - values. If an operation does not have directives specified, then the - directives of the singleton group containing that operation are used. To - prevent this, set the directives to an empty dictionary for that - operation. - :type operation_directives: - dict - :param options: + resources. Operation names are keys and the dictionaries of directives + are values. If an operation does not have directives specified, then + the directives of the singleton group containing that operation are + used. To prevent this, set the directives to an empty dictionary for + that operation. + options : str A string of options to append to the output of the object's call method. - This lets options like ``--num_passes`` to be given to a group. - :type options: - str + This allows options like ``--num_passes`` to be given to a group. + """ MAX_LEN_ID = 100 @@ -965,38 +961,40 @@ def __repr__(self): def _eligible(self, jobs, ignore_conditions=IgnoreConditions.NONE): """Determine if at least one operation is eligible. - A FlowGroup is eligible for execution if at least one of its + A :class:`~.FlowGroup` is eligible for execution if at least one of its associated operations is eligible. - :param jobs: + Parameters + ---------- + jobs : tuple The signac job handles. - :type jobs: - tuple - :param ignore_conditions: - Specify if pre and/or post conditions are to be ignored while - checking eligibility. The default is + ignore_conditions : :class:`~.IgnoreConditions` + Specify if preconditions and/or postconditions are to be ignored + while checking eligibility. The default is :class:`IgnoreConditions.NONE`. - :type ignore_conditions: - :class:`~.IgnoreConditions` - :return: + + Returns + ------- + bool Whether the group is eligible. - :rtype: - bool + """ return any(op._eligible(jobs, ignore_conditions) for op in self) def _complete(self, jobs): - """Check if post-conditions are met for all operations in the group. + """Check if postconditions are met for all operations in the group. - :param jobs: + Parameters + ---------- + jobs : tuple The signac job handles. - :type jobs: - tuple - :return: + + Returns + ------- + bool Whether the group is complete (all contained operations are complete). - :rtype: - bool + """ return all(op._complete(jobs) for op in self) @@ -1007,75 +1005,95 @@ def eligible(self, job, ignore_conditions=IgnoreConditions.NONE): A FlowGroup is eligible for execution if at least one of its associated operations is eligible. - :param job: + Parameters + ---------- + job : :class:`~signac.contrib.job.Job` A :class:`~signac.contrib.job.Job` from the signac workspace. - :type job: - :class:`~signac.contrib.job.Job` - :param ignore_conditions: - Specify if pre and/or post conditions are to be ignored while checking eligibility. - The default is :class:`IgnoreConditions.NONE`. - :type ignore_conditions: - :class:`~.IgnoreConditions` - :return: + ignore_conditions : :class:`~.IgnoreConditions` + Specify if preconditions and/or postconditions are to be ignored + while checking eligibility. The default is + :class:`IgnoreConditions.NONE`. + + Returns + ------- + bool Whether the group is eligible. - :rtype: - bool + """ return self._eligible((job,), ignore_conditions) @deprecated(deprecated_in="0.11", removed_in="0.13", current_version=__version__) def complete(self, job): - """Check if all BaseFlowOperation post-conditions are met. + """Check if all BaseFlowOperation postconditions are met. - :param job: + Parameters + ---------- + job : :class:`~signac.contrib.job.Job` A :class:`~signac.contrib.job.Job` from the signac workspace. - :type job: - :class:`~signac.contrib.job.Job` - :return: + + Returns + ------- + bool Whether the group is complete (all contained operations are complete). - :rtype: - bool + """ return self._complete((job,)) def add_operation(self, name, operation, directives=None): - """Add an operation to the FlowGroup. + """Add an operation to the :class:`~.FlowGroup`. - :param name: + Parameters + ---------- + name : str The name of the operation. - :type name: - str - :param operation: - The workflow operation to add to the FlowGroup. - :type operation: - :class:`BaseFlowOperation` - :param directives: - The operation specific directives. - :type directives: - dict + operation : :class:`BaseFlowOperation` + The workflow operation to add to the :class:`~.FlowGroup`. + directives : dict + The operation specific directives. (Default value = None) + """ self.operations[name] = operation if directives is not None: self.operation_directives[name] = directives def isdisjoint(self, group): - """Return whether two groups are disjoint (do not share any common operations). + """Return whether two groups are disjoint. - :param group: + Groups are disjoint if they do not share any common operations. + + Parameters + ---------- + group : :class:`flow.project.FlowGroup` The other FlowGroup to compare to. - :type group: - :class:`flow.project.FlowGroup` - :return: + + Returns + ------- + bool Returns ``True`` if ``group`` and ``self`` share no operations, otherwise returns ``False``. - :rtype: - bool + """ return set(self).isdisjoint(set(group)) def _generate_id(self, jobs, operation_name=None, index=0): - """Generate a unique id which identifies this group and job(s).""" + """Generate a unique id which identifies this group and job(s). + + Parameters + ---------- + jobs : sequence of :class:`signac.contrib.job.Job` + Jobs defining the unique id. + operation_name : str + Operation name defining the unique id. (Default value = None) + index : int + Index for the :class:`~._JobOperation`. (Default value = 0) + + Returns + ------- + str + The unique id. + + """ project = jobs[0]._project # The full name is designed to be truly unique for each job-group. @@ -1139,42 +1157,32 @@ def _create_submission_job_operation( Creates a _JobOperation for use in submitting and scripting. - :param entrypoint: + Parameters + ---------- + entrypoint : dict The path and executable, if applicable, to point to for execution. - :type entrypoint: - dict - :param default_directives: - The default directives to use for the operations. This is to allow for user specified - groups to 'inherit' directives from ``default_directives``. If no defaults are desired, - the argument can be set to an empty dictionary. This must be done explicitly, however. - :type default_directives: - dict - :param jobs: + default_directives : dict + The default directives to use for the operations. This is to allow + for user specified groups to 'inherit' directives from + ``default_directives``. If no defaults are desired, the argument + can be set to an empty dictionary. This must be done explicitly, + however. + jobs : tuple of :class:`~signac.contrib.job.Job` The jobs that the :class:`~._JobOperation` is based on. - :type jobs: - tuple of :class:`~signac.contrib.job.Job` - :param ignore_conditions: - Specify if pre and/or post conditions are to be ignored while - checking eligibility. The default is - :class:`IgnoreConditions.NONE`. - :type ignore_conditions: - :class:`~.IgnoreConditions` - :param ignore_conditions_on_execution: - Specify if pre and/or post conditions are to be ignored while - checking eligibility during execution (after submission). The + ignore_conditions_on_execution : :class:`~.IgnoreConditions` + Specify if preconditions and/or postconditions are to be ignored + while checking eligibility during execution (after submission). The default is :class:`IgnoreConditions.NONE`. - :type ignore_conditions_on_execution: - :class:`~.IgnoreConditions` - :param index: - Index for the :class:`~._JobOperation`. - :type index: - int - :return: - Returns a :class:`~._SubmissionJobOperation` for submitting the group. The - :class:`~._JobOperation` will have directives that have been collected - appropriately from its contained operations. - :rtype: - :class:`_SubmissionJobOperation` + index : int + Index for the :class:`~._JobOperation`. (Default value = 0) + + Returns + ------- + :class:`_SubmissionJobOperation` + Returns a :class:`~._SubmissionJobOperation` for submitting the + group. The :class:`~._JobOperation` will have directives that have + been collected appropriately from its contained operations. + """ unevaluated_cmd = functools.partial( self._submit_cmd, @@ -1189,6 +1197,20 @@ def _get_run_ops(ignore_ops, additional_ignores_flag): Returns operations that match the combination of the conditions required by ``_create_submission_job_operation`` and the ignored flags, and remove operations in the ``ignore_ops`` list. + + Parameters + ---------- + ignore_ops : iterable + Operations to ignore. + additional_ignores_flag : :class:`~.IgnoreConditions` + An additional set of ignore flags combined with the ignore + flags used for execution. + + Returns + ------- + list of :class:`_JobOperation` + Runnable operations. + """ return list( set( @@ -1236,30 +1258,32 @@ def _create_run_job_operations( ): """Create _JobOperation object(s) from the FlowGroup. - Yields a _JobOperation for each contained operation given proper conditions are met. + Yields a _JobOperation for each contained operation given proper + conditions are met. - :param entrypoint: + Parameters + ---------- + entrypoint : dict The path and executable, if applicable, to point to for execution. - :type entrypoint: - dict - :param default_directives: - The default directives to use for the operations. This is to allow for user specified - groups to 'inherent' directives from ``default_directives``. If no defaults are desired, - the argument must be explicitly set to an empty dictionary. - :type default_directives: - dict - :param jobs: + default_directives : dict + The default directives to use for the operations. This is to allow + for user-specified groups to inherit directives from + ``default_directives``. If no defaults are desired, the argument + must be explicitly set to an empty dictionary. + jobs : tuple of :class:`~signac.contrib.job.Job` The jobs that the :class:`~._JobOperation` is based on. - :type jobs: - tuple of :class:`~signac.contrib.job.Job` - :param index: - Index for the :class:`~._JobOperation`. - :type index: - int - :return: - Returns an iterator over eligible :class:`~._JobOperation`s. - :rtype: - Iterator[_JobOperation] + ignore_conditions : :class:`~.IgnoreConditions` + Specify if preconditions and/or postconditions are to be ignored + when determining eligibility check. The default is + :class:`IgnoreConditions.NONE`. + index : int + Index for the :class:`~._JobOperation`. (Default value = 0) + + Returns + ------- + Iterator[_JobOperation] + Iterator of eligible instances of :class:`~._JobOperation`. + """ # Assuming all the jobs belong to the same FlowProject env = jobs[0]._project._environment @@ -1321,31 +1345,33 @@ class _FlowProjectClass(type): def __new__(metacls, name, bases, namespace): cls = type.__new__(metacls, name, bases, dict(namespace)) - # All operation functions are registered with the operation() classmethod, which is - # intended to be used as a decorator function. _OPERATION_FUNCTIONS is a list of tuples - # of the operation name and the operation function. In addition, pre and post conditions + # All operation functions are registered with the operation() + # classmethod, which is intended to be used as a decorator function. + # _OPERATION_FUNCTIONS is a list of tuples of the operation name and + # the operation function. In addition, preconditions and postconditions # are registered with the class. cls._OPERATION_FUNCTIONS = [] cls._OPERATION_PRE_CONDITIONS = defaultdict(list) cls._OPERATION_POST_CONDITIONS = defaultdict(list) - # All label functions are registered with the label() classmethod, which is intended - # to be used as decorator function. The _LABEL_FUNCTIONS dict contains the function as - # key and the label name as value, or None to use the default label name. + # All label functions are registered with the label() classmethod, + # which is intended to be used as decorator function. The + # _LABEL_FUNCTIONS dict contains the function as key and the label name + # as value, or None to use the default label name. cls._LABEL_FUNCTIONS = {} - # Give the class a pre and post class that are aware of the class they - # are in. + # Give the class a preconditions and post class that are aware of the + # class they are in. cls.pre = cls._setup_pre_conditions_class(parent_class=cls) cls.post = cls._setup_post_conditions_class(parent_class=cls) - # All groups are registered with the function returned by the make_group - # classmethod. In contrast to operations and labels, the + # All groups are registered with the function returned by the + # make_group classmethod. In contrast to operations and labels, the # make_group classmethod does not serve as the decorator, the functor - # it returns does. The _GROUPS list records the groups created and their - # passed parameters for later initialization. The _GROUP_NAMES set stores - # whether a group name has already been used. + # it returns does. The _GROUPS list records the groups created and + # their passed parameters for later initialization. The _GROUP_NAMES + # set stores whether a group name has already been used. cls._GROUPS = [] cls._GROUP_NAMES = set() @@ -1354,9 +1380,9 @@ def __new__(metacls, name, bases, namespace): @staticmethod def _setup_pre_conditions_class(parent_class): class pre(_condition): - """Define and evaluate pre-conditions for operations. + """Define and evaluate preconditions for operations. - A pre-condition is a function accepting one or more jobs as + A precondition is a function accepting one or more jobs as positional arguments (``*jobs``) that must evaluate to True for this operation to be eligible for execution. For example: @@ -1368,8 +1394,8 @@ def hello(job): print('hello', job) job.doc.hello = True - The *hello* operation would only execute if the 'hello' key in the job - document does not evaluate to True. + The *hello* operation would only execute if the 'hello' key in the + job document does not evaluate to True. An optional tag may be associated with the condition. These tags are used by :meth:`~.detect_operation_graph` when comparing @@ -1395,7 +1421,7 @@ def __call__(self, func): @classmethod def copy_from(cls, *other_funcs): - """Copy pre-conditions from other operation(s). + """Copy preconditions from other operation(s). True if and only if all pre conditions of other operation function(s) are met. @@ -1408,9 +1434,9 @@ def copy_from(cls, *other_funcs): @classmethod def after(cls, *other_funcs): - """Pre-condition to run an operation after other operations. + """Precondition to run an operation after other operations. - True if and only if all post conditions of other operation + True if and only if all postconditions of other operation function(s) are met. """ operation_functions = [ @@ -1432,9 +1458,9 @@ def after(cls, *other_funcs): @staticmethod def _setup_post_conditions_class(parent_class): class post(_condition): - """Define and evaluate post-conditions for operations. + """Define and evaluate postconditions for operations. - A post-condition is a function accepting one or more jobs as + A postcondition is a function accepting one or more jobs as positional arguments (``*jobs``) that must evaluate to True for this operation to be considered complete. For example: @@ -1446,8 +1472,9 @@ def bye(job): print('bye', job) job.doc.bye = True - The *bye* operation would be considered complete and therefore no longer - eligible for execution once the 'bye' key in the job document evaluates to True. + The *bye* operation would be considered complete and therefore no + longer eligible for execution once the 'bye' key in the job + document evaluates to True. An optional tag may be associated with the condition. These tags are used by :meth:`~.detect_operation_graph` when comparing @@ -1473,9 +1500,9 @@ def __call__(self, func): @classmethod def copy_from(cls, *other_funcs): - """Copy post-conditions from other operation(s). + """Copy postconditions from other operation(s). - True if and only if all post conditions of other operation + True if and only if all postconditions of other operation function(s) are met. """ return cls( @@ -1505,24 +1532,21 @@ def hello(job): FlowProject().main() - :param config: + Parameters + ---------- + config : :class:`signac.contrib.project._ProjectConfig` A signac configuration, defaults to the configuration loaded from the current directory. - :type config: - :class:`signac.contrib.project._ProjectConfig` - :param environment: + environment : :class:`flow.environment.ComputeEnvironment` An environment to use for scheduler submission. If ``None``, the environment is automatically identified. The default is ``None``. - :type environment: - :class:`flow.environment.ComputeEnvironment` - :param entrypoint: + entrypoint : dict A dictionary with two possible keys: ``'executable'`` and ``'path'``. The path represents the location of the script file (the script file must call :meth:`FlowProject.main`). The executable represents the location of the Python interpreter used for the execution of :class:`~.BaseFlowOperation` that are Python functions. - :type entrypoint: - dict + """ def __init__(self, config=None, environment=None, entrypoint=None): @@ -1562,9 +1586,9 @@ def __init__(self, config=None, environment=None, entrypoint=None): def _setup_template_environment(self): """Set up the jinja2 template environment. - The templating system is used to generate templated scripts for the script() - and _submit_operations() / submit() function and the corresponding command line - subcommands. + The templating system is used to generate templated scripts for the + script() and _submit_operations() / submit() function and the + corresponding command line subcommands. """ if self._config.get("flow") and self._config["flow"].get("environment_modules"): envs = self._config["flow"].as_list("environment_modules") @@ -1670,8 +1694,8 @@ def foo_label(job): if job.document.get('foo', False): return 'foo-label-text' - The ``foo-label-text`` label will now show up in the status view for each job, - where the ``foo`` key evaluates true. + The ``foo-label-text`` label will now show up in the status view for + each job, where the ``foo`` key evaluates true. If the label functions returns any type other than ``str``, the label name will be the name of the function if and only if the return value @@ -1686,16 +1710,26 @@ def foo_label(job): Finally, specify a label name by providing it as the first argument to the ``label()`` decorator. - :param label_name_or_func: - A label name or callable. - :type label_name_or_func: - str or callable + Parameters + ---------- + label_name_or_func : str or callable + A label name or callable. (Default value = None) + + Returns + ------- + callable + A decorator for the label function. + """ if callable(label_name_or_func): + # This handles the case where no label name is given, as in + # @FlowProject.label. label_name_or_func is a function. cls._LABEL_FUNCTIONS[label_name_or_func] = None return label_name_or_func def label_func(func): + # This handles the case where a label name is given, as in + # @FlowProject.label("label_name"). label_name_or_func is a string. cls._LABEL_FUNCTIONS[func] = label_name_or_func return func @@ -1705,9 +1739,9 @@ def detect_operation_graph(self): """Determine the directed acyclic graph given by operation conditions. In general, executing a given operation registered with a FlowProject - just involves checking the operation's pre- and post-conditions to - determine eligibility. More generally, however, the pre- and - post-conditions define a directed acyclic graph that governs the + just involves checking the operation's preconditions and postconditions + to determine eligibility. More generally, however, the preconditions + and postconditions define a directed acyclic graph that governs the execution of all operations. Visualizing this graph can be useful for finding logic errors in the specified conditions, and having this graph computed also enables additional execution modes. For example, using @@ -1716,18 +1750,18 @@ def detect_operation_graph(self): executing all necessary operations can be automated. The graph is determined by iterating over all pairs of operations and - checking for equality of pre- and post-conditions. The algorithm builds - an adjacency matrix based on whether the pre-conditions for one - operation match the post-conditions for another. The comparison of - operations is conservative; by default, conditions must be composed of - identical code to be identified as equal (technically, they must be - bytecode equivalent, i.e. ``cond1.__code__.co_code == + checking for equality of preconditions and postconditions. The + algorithm builds an adjacency matrix based on whether the preconditions + for one operation match the postconditions for another. The comparison + of operations is conservative; by default, conditions must be composed + of identical code to be identified as equal (technically, they must + have equivalent bytecode, i.e. ``cond1.__code__.co_code == cond2.__code__.co_code``). Users can specify that conditions should be treated as equal by providing tags to the operations. - Given a FlowProject subclass defined in a module ``project.py``, the - output graph could be visualized using Matplotlib and NetworkX with the - following code: + Given a :class:`~.FlowProject` subclass defined in a module + ``project.py``, the output graph could be visualized using Matplotlib + and NetworkX with the following code: .. code-block:: python @@ -1752,11 +1786,19 @@ def detect_operation_graph(self): plt.show() - Raises a ``RuntimeError`` if a condition does not have a tag. This can - occur when using ``functools.partial``, and a manually specified - condition tag has not been set. + Returns + ------- + list of lists of int + The adjacency matrix of operation dependencies. A zero indicates no + dependency, and a 1 indicates dependency. This can be converted to + a graph using NetworkX. - :raises: RuntimeError + Raises + ------ + :class:`RuntimeError` + If a condition does not have a tag. This can occur when using + ``functools.partial``, and a manually specified condition tag has + not been set. """ @@ -1851,7 +1893,19 @@ def _register_labels(self): @classmethod def _alias(cls, name): - """Use alias if specified.""" + """Use alias if specified. + + Parameters + ---------- + name : str + Long name to abbreviate. + + Returns + ------- + str + Abbreviation if it exists, otherwise the input name. + + """ try: return abbreviate(name, cls.ALIASES.get(name, name)) except TypeError: @@ -1870,14 +1924,16 @@ def _store_bundled(self, operations): operation will not be stored, but instead the operation's id is directly returned. - :param operations: + Parameters + ---------- + operations : A sequence of instances of :class:`._JobOperation` The operations to bundle. - :type operations: - A sequence of instances of :class:`._JobOperation` - :return: + + Returns + ------- + str The bundle id. - :rtype: - str + """ if len(operations) == 1: return operations[0].id @@ -1906,23 +1962,45 @@ def _expand_bundled_jobs(self, scheduler_jobs): def scheduler_jobs(self, scheduler): """Fetch jobs from the scheduler. - This function will fetch all scheduler jobs from the scheduler - and also expand bundled jobs automatically. + This function will fetch all scheduler jobs from the scheduler and also + expand bundled jobs automatically. - However, this function will not automatically filter scheduler - jobs which are not associated with this project. + However, this function will not automatically filter scheduler jobs + which are not associated with this project. - :param scheduler: + Parameters + ---------- + scheduler : :class:`~.Scheduler` The scheduler instance. - :type scheduler: - :class:`~.flow.manage.Scheduler` - :yields: - All scheduler jobs fetched from the scheduler instance. + + Yields + ------ + :class:`~.ClusterJob`: + All cluster jobs fetched from the scheduler. + """ yield from self._expand_bundled_jobs(scheduler.jobs()) def _get_operations_status(self, jobs, cached_status): - """Return a dict with information about job-operations for this aggregate.""" + """Return a dict with information about job-operations for this aggregate. + + Parameters + ---------- + jobs : :class:`~signac.contrib.job.Job` or aggregate of jobs + The signac job or aggregate. + cached_status : dict + Dictionary of cached status information. The keys are uniquely + generated ids for each group and job. The values are instances of + :class:`~.JobStatus`. + + Yields + ------ + str + Operation name. + dict + Operation status dictionary. + + """ starting_dict = functools.partial(dict, scheduler_status=JobStatus.unknown) status_dict = defaultdict(starting_dict) operation_names = list(self.operations.keys()) @@ -1947,24 +2025,22 @@ def _get_operations_status(self, jobs, cached_status): def get_job_status(self, job, ignore_errors=False, cached_status=None): """Return status information about a job. - :param job: + Parameters + ---------- + job : :class:`~signac.contrib.job.Job` The signac job. - :type job: - :class:`~signac.contrib.job.Job` - :param ignore_errors: - Whether to ignore exceptions raised during status check. - :type ignore_errors: - bool - :param cached_status: + ignore_errors : bool + Whether to ignore exceptions raised during status check. (Default value = False) + cached_status : dict Dictionary of cached status information. The keys are uniquely generated ids for each group and job. The values are instances of - :class:`~.JobStatus`. - :type cached_status: - dict - :returns: + :class:`~.JobStatus`. (Default value = None) + + Returns + ------- + dict A dictionary containing job status for all jobs. - :rtype: - dict + """ # TODO: Add support for aggregates for this method. result = {} @@ -2002,17 +2078,16 @@ def get_job_status(self, job, ignore_errors=False, cached_status=None): def _fetch_scheduler_status(self, jobs=None, file=None, ignore_errors=False): """Update the status docs. - :param jobs: - The signac job or aggregate. - :type jobs: - sequence of :class:`~signac.contrib.job.Job` or aggregates of jobs - :param file: + Parameters + ---------- + jobs : sequence of :class:`~signac.contrib.job.Job` or aggregates of jobs + The signac job or aggregate. (Default value = None) + file : file-like object File where status information is printed. If ``None``, ``sys.stderr`` is used. The default is ``None``. - :param ignore_errors: - Whether to ignore exceptions raised during status check. - :type ignore_errors: - bool + ignore_errors : bool + Whether to ignore exceptions raised during status check. (Default value = False) + """ if file is None: file = sys.stderr @@ -2061,20 +2136,22 @@ def _get_group_status(self, group_name, ignore_errors=False, cached_status=None) Status information is fetched for all jobs/aggregates associated with this group and returned as a dict. - :param ignore_errors: - Whether to ignore exceptions raised during status check. - :type ignore_errors: - bool - :param cached_status: + Parameters + ---------- + group_name : str + Group name. + ignore_errors : bool + Whether to ignore exceptions raised during status check. (Default value = False) + cached_status : dict Dictionary of cached status information. The keys are uniquely generated ids for each group and job. The values are instances of - :class:`~.JobStatus`. - :type cached_status: - dict - :returns: + :class:`~.JobStatus`. (Default value = None) + + Returns + ------- + dict A dictionary containing job status for all jobs. - :rtype: - dict + """ group = self._groups[group_name] status_dict = {} @@ -2118,7 +2195,21 @@ def _get_group_status(self, group_name, ignore_errors=False, cached_status=None) } def _get_job_labels(self, job, ignore_errors=False): - """Return a dict with information about the labels of a job.""" + """Return a dict with information about the labels of a job. + + Parameters + ---------- + job : :class:`signac.contrib.job.Job` + Job handle. + ignore_errors : bool + Whether to ignore errors raised while fetching labels. (Default value = False) + + Returns + ------- + dict + Dictionary with keys ``job_id``, ``labels``, and ``_labels_error``. + + """ result = {} result["job_id"] = str(job) try: @@ -2144,30 +2235,28 @@ def _fetch_status( ): """Fetch status for the provided aggregates / jobs. - :param aggregates: + Parameters + ---------- + aggregates : list The aggregates for which a user requested to fetch status. - :type aggregates: - list - :param distinct_jobs: + distinct_jobs : list of :class:`~signac.contrib.job.Job` Distinct jobs fetched from the ids provided in the ``jobs`` argument. This is used for fetching labels for a job because a label is not associated with an aggregate. - :type distinct_jobs: - list of :class:`~signac.contrib.job.Job` - :param ignore_errors: + err : file-like object + File where status information is printed. + ignore_errors : bool Fetch status even if querying the scheduler fails. - :type ignore_errors: - bool - :param status_parallelization: + status_parallelization : str Parallelization mode for fetching the status. By default, thread parallelism is used. - :type status_parallelization: - str - :returns: + + Returns + ------- + list A list of dictionaries containing job ids, operations, labels, and any errors caught. - :rtype: - list + """ # The argument status_parallelization is used so that _fetch_status method # gets to know whether the deprecated argument no_parallelization passed @@ -2440,91 +2529,60 @@ def print_status( ): """Print the status of the project. - :param jobs: - Only execute operations for the given jobs, or all if the argument is omitted. - :type jobs: - Sequence of instances of :class:`~signac.contrib.job.Job`. - :param overview: - Aggregate an overview of the project' status. - :type overview: - bool - :param overview_max_lines: - Limit the number of overview lines. - :type overview_max_lines: - int - :param detailed: - Print a detailed status of each job. - :type detailed: - bool - :param parameters: - Print the value of the specified parameters. - :type parameters: - list of str - :param param_max_width: - Limit the number of characters of parameter columns. - :type param_max_width: - int - :param expand: - Present labels and operations in two separate tables. - :type expand: - bool - :param all_ops: - Include operations that are not eligible to run. - :type all_ops: - bool - :param only_incomplete: - Only show jobs that have eligible operations. - :type only_incomplete: - bool - :param dump_json: + Parameters + ---------- + jobs : Sequence of instances of :class:`~signac.contrib.job.Job`. + Only execute operations for the given jobs, or all if the argument + is omitted. (Default value = None) + overview : bool + Aggregate an overview of the project' status. (Default value = True) + overview_max_lines : int + Limit the number of overview lines. (Default value = None) + detailed : bool + Print a detailed status of each job. (Default value = False) + parameters : list of str + Print the value of the specified parameters. (Default value = None) + param_max_width : int + Limit the number of characters of parameter columns. (Default value = None) + expand : bool + Present labels and operations in two separate tables. (Default value = False) + all_ops : bool + Include operations that are not eligible to run. (Default value = False) + only_incomplete : bool + Only show jobs that have eligible operations. (Default value = False) + dump_json : bool Output the data as JSON instead of printing the formatted output. - :type dump_json: - bool - :param unroll: - Separate columns for jobs and the corresponding operations. - :type unroll: - bool - :param compact: - Print a compact version of the output. - :type compact: - bool - :param pretty: - Prettify the output. - :type pretty: - bool - :param file: + (Default value = False) + unroll : bool + Separate columns for jobs and the corresponding operations. (Default value = True) + compact : bool + Print a compact version of the output. (Default value = False) + pretty : bool + Prettify the output. (Default value = False) + file : str Redirect all output to this file, defaults to sys.stdout. - :type file: - str - :param err: + err : str Redirect all error output to this file, defaults to sys.stderr. - :type err: - str - :param ignore_errors: - Print status even if querying the scheduler fails. - :type ignore_errors: - bool - :param template: - User provided Jinja2 template file. - :type template: - str - :param profile: - Show profile result. - :type profile: - bool - :param eligible_jobs_max_lines: - Limit the number of operations and its eligible job count printed in the overview. - :type eligible_jobs_max_lines: - int - :param output_format: + ignore_errors : bool + Print status even if querying the scheduler fails. (Default value = False) + template : str + User provided Jinja2 template file. (Default value = None) + profile : bool + Show profile result. (Default value = False) + eligible_jobs_max_lines : int + Limit the number of operations and its eligible job count printed + in the overview. (Default value = None) + output_format : str Status output format, supports: 'terminal' (default), 'markdown' or 'html'. - :type output_format: - str - :return: + no_parallelize : + (Default value = False) + + Returns + ------- + :class:`~.Renderer` A Renderer class object that contains the rendered string. - :rtype: - :class:`~.Renderer` + """ if file is None: file = sys.stdout @@ -2875,29 +2933,24 @@ def _run_operations( ): """Execute the next operations as specified by the project's workflow. - See also: :meth:`~.run` + See also: :meth:`~.run`. + + Parameters + ---------- + operations : Sequence of instances of :class:`._JobOperation` + The operations to execute (optional). (Default value = None) + pretend : bool + Do not actually execute the operations, but show the commands that + would have been executed. (Default value = False) + np : int + The number of processors to use for each operation. (Default value = None) + timeout : int + An optional timeout for each operation in seconds after which + execution will be cancelled. Use -1 to indicate no timeout (the + default). + progress : bool + Show a progress bar during execution. (Default value = False) - :param operations: - The operations to execute (optional). - :type operations: - Sequence of instances of :class:`._JobOperation` - :param pretend: - Do not actually execute the operations, but show which command would have been used. - :type pretend: - bool - :param np: - The number of processors to use for each operation. - :type np: - int - :param timeout: - An optional timeout for each operation in seconds after which execution will - be cancelled. Use -1 to indicate not timeout (the default). - :type timeout: - int - :param progress: - Show a progress bar during execution. - :type progress: - bool """ if timeout is not None and timeout < 0: timeout = None @@ -2961,27 +3014,22 @@ def run_operations( See also: :meth:`~.run` - :param operations: - The operations to execute (optional). - :type operations: - Sequence of instances of :class:`.JobOperation` - :param pretend: - Do not actually execute the operations, but show which command would have been used. - :type pretend: - bool - :param np: - The number of processors to use for each operation. - :type np: - int - :param timeout: - An optional timeout for each operation in seconds after which execution will - be cancelled. Use -1 to indicate not timeout (the default). - :type timeout: - int - :param progress: - Show a progress bar during execution. - :type progress: - bool + Parameters + ---------- + operations : Sequence of instances of :class:`.JobOperation` + The operations to execute (optional). (Default value = None) + pretend : bool + Do not actually execute the operations, but show the commands that + would have been executed. (Default value = False) + np : int + The number of processors to use for each operation. (Default value = None) + timeout : int + An optional timeout for each operation in seconds after which + execution will be cancelled. Use -1 to indicate no timeout (the + default). + progress : bool + Show a progress bar during execution. (Default value = False) + """ return self._run_operations(operations, pretend, np, timeout, progress) @@ -3008,11 +3056,13 @@ def _job_operation_from_tuple(self, data): def _run_operations_in_parallel(self, pool, pickle, operations, progress, timeout): """Execute operations in parallel. - This function executes the given list of operations with the provided process pool. + This function executes the given list of operations with the provided + process pool. - Since pickling of the project instance is likely to fail, we manually pickle the - project instance and the operations before submitting them to the process pool to - enable us to try different pool and pickle module combinations. + Since pickling of the project instance is likely to fail, we manually + pickle the project instance and the operations before submitting them + to the process pool to enable us to try different pool and pickle + module combinations. """ try: serialized_root = pickle.dumps(self.root_directory()) @@ -3099,62 +3149,48 @@ def run( are executed, unless it reaches the maximum number of passes per operation or the maximum number of executions. - By default there is no limit on the total number of executions, but a specific - operation will only be executed once per job. This is to avoid accidental - infinite loops when no or faulty post conditions are provided. + By default there is no limit on the total number of executions, but a + specific operation will only be executed once per job. This is to avoid + accidental infinite loops when no or faulty postconditions are + provided. - :param jobs: + Parameters + ---------- + jobs : iterable of :class:`~signac.contrib.job.Job` or aggregates of jobs. Only execute operations for the given jobs or aggregates of jobs, - or all if the argument is omitted. - :type jobs: - Sequence of instances of :class:`~signac.contrib.job.Job` or - sequence of aggregates where each aggregate is a sequence - of :class:`~signac.contrib.job.Job`. - :param names: - Only execute operations that are in the provided set of names, or all, if the - argument is omitted. - :type names: - Sequence of :class:`str` - :param pretend: - Do not actually execute the operations, but show which command would have been used. - :type pretend: - bool - :param np: + or all if the argument is omitted. (Default value = None) + names : iterable of :class:`str` + Only execute operations that are in the provided set of names, or + all if the argument is omitted. (Default value = None) + pretend : bool + Do not actually execute the operations, but show the commands that + would have been executed. (Default value = False) + np : int Parallelize to the specified number of processors. Use -1 to - parallelize to all available processing units. - :type np: - int - :param timeout: + parallelize to all available processing units. (Default value = None) + timeout : int An optional timeout for each operation in seconds after which execution will be cancelled. Use -1 to indicate no timeout (the default). - :type timeout: - int - :param num: - The total number of operations that are executed will not exceed this argument - if provided. - :type num: - int - :param num_passes: + num : int + The total number of operations that are executed will not exceed + this argument if provided. (Default value = None) + num_passes : int or None The total number of executions of one specific job-operation pair will not exceed this argument. The default is 1, there is no limit if this argument is None. - :type num_passes: - int or None - :param progress: - Show a progress bar during execution. - :type progress: - bool - :param order: + progress : bool + Show a progress bar during execution. (Default value = False) + order : str, callable, or None Specify the order of operations. Possible values are: - * 'none' or None (no specific order) - * 'by-job' (operations are grouped by job) - * 'by-op' (operations are grouped by operation) - * 'cyclic' (order operations cyclic by job) - * 'random' (shuffle the execution order randomly) - * callable (a callable returning a comparison key for an - operation used to sort operations) + * 'none' or None (no specific order) + * 'by-job' (operations are grouped by job) + * 'by-op' (operations are grouped by operation) + * 'cyclic' (order operations cyclic by job) + * 'random' (shuffle the execution order randomly) + * callable (a callable returning a comparison key for an + operation used to sort operations) The default value is ``'none'``, which is equivalent to ``'by-op'`` in the current implementation. @@ -3162,18 +3198,16 @@ def run( .. note:: Users are advised to not rely on a specific execution order as - a substitute for defining the workflow in terms of pre- and - post-conditions. However, a specific execution order may be - more performant in cases where operations need to access and - potentially lock shared resources. - - :type order: - str, callable, or None - :param ignore_conditions: - Specify if pre and/or post conditions check is to be ignored for - eligibility check. The default is :class:`IgnoreConditions.NONE`. - :type ignore_conditions: - :class:`~.IgnoreConditions` + a substitute for defining the workflow in terms of + preconditions and postconditions. However, a specific execution + order may be more performant in cases where operations need to + access and potentially lock shared resources. + + ignore_conditions : :class:`~.IgnoreConditions` + Specify if preconditions and/or postconditions are to be ignored + when determining eligibility. The default is + :class:`IgnoreConditions.NONE`. + """ aggregates = self._convert_aggregates_from_jobs(jobs) @@ -3229,13 +3263,13 @@ def select(operation): f"passes ({num_passes})." ) - # Warn if an operation has no post-conditions set. + # Warn if an operation has no postconditions set. has_post_conditions = len( self.operations[operation.name]._postconditions ) if not has_post_conditions: log( - f"Operation '{operation.name}' has no post-conditions!", + f"Operation '{operation.name}' has no postconditions!", logging.WARNING, ) @@ -3469,22 +3503,22 @@ def _script( ): """Generate a run script to execute given operations. - :param operations: + Parameters + ---------- + operations : Sequence of instances of :class:`._JobOperation` The operations to execute. - :type operations: - Sequence of instances of :class:`._JobOperation` - :param parallel: + parallel : bool Execute all operations in parallel (default is False). - :type parallel: - bool - :param template: - The name of the template to use to generate the script. - :type template: - str - :param show_template_help: - Show help related to the templating system and then exit. - :type show_template_help: - bool + template : str + The name of the template to use to generate the script. (Default value = "script.sh") + show_template_help : bool + Show help related to the templating system and then exit. (Default value = False) + + Returns + ------- + str + Rendered template of run script. + """ template_environment = self._template_environment() template = template_environment.get_template(template) @@ -3504,22 +3538,22 @@ def script( ): """Generate a run script to execute given operations. - :param operations: + Parameters + ---------- + operations : Sequence of instances of :class:`~.JobOperation` The operations to execute. - :type operations: - Sequence of instances of :class:`.JobOperation` - :param parallel: + parallel : bool Execute all operations in parallel (default is False). - :type parallel: - bool - :param template: - The name of the template to use to generate the script. - :type template: - str - :param show_template_help: - Show help related to the templating system and then exit. - :type show_template_help: - bool + template : str + The name of the template to use to generate the script. (Default value = "script.sh") + show_template_help : bool + Show help related to the templating system and then exit. (Default value = False) + + Returns + ------- + str + Rendered template of run script. + """ return self._script(operations, parallel, template, show_template_help) @@ -3566,43 +3600,38 @@ def _submit_operations( ): r"""Submit a sequence of operations to the scheduler. - :param operations: + Parameters + ---------- + operations : A sequence of instances of :class:`._JobOperation` The operations to submit. - :type operations: - A sequence of instances of :class:`._JobOperation` - :param _id: - The _id to be used for this submission. - :type _id: - str - :param parallel: - Execute all bundled operations in parallel. - :type parallel: - bool - :param flags: - Additional options to be forwarded to the scheduler. - :type flags: - list - :param force: - Ignore all warnings or checks during submission, just submit. - :type force: - bool - :param template: - The name of the template file to be used to generate the submission script. - :type template: - str - :param pretend: + _id : str + The _id to be used for this submission. (Default value = None) + parallel : bool + Execute all bundled operations in parallel. (Default value = False) + flags : list + Additional options to be forwarded to the scheduler. (Default value = None) + force : bool + Ignore all warnings or checks during submission, just submit. (Default value = False) + template : str + The name of the template file to be used to generate the submission + script. (Default value = "script.sh") + pretend : bool Do not actually submit, but only print the submission script to screen. Useful - for testing the submission workflow. - :type pretend: - bool - :param show_template_help: - Show information about available template variables and filters and exit. - :type show_template_help: - bool - :param \*\*kwargs: - Additional keyword arguments to be forwarded to the scheduler. - :return: + for testing the submission workflow. (Default value = False) + show_template_help : bool + Show information about available template variables and filters and + exit. (Default value = False) + env : :class:`~.ComputeEnvironment` + The environment to use for submission. Uses the environment defined + by the :class:`~.FlowProject` if None (Default value = None). + \*\*kwargs : + Additional keyword aruguments forwarded to :meth:`~.ComputeEnvironment.submit`. + + Returns + ------- + :class:`~.JobStatus` or None Returns the submission status after successful submission or None. + """ if _id is None: _id = self._store_bundled(operations) @@ -3683,43 +3712,38 @@ def submit_operations( ): r"""Submit a sequence of operations to the scheduler. - :param operations: + Parameters + ---------- + operations : A sequence of instances of :class:`.JobOperation` The operations to submit. - :type operations: - A sequence of instances of :class:`.JobOperation` - :param _id: - The _id to be used for this submission. - :type _id: - str - :param parallel: - Execute all bundled operations in parallel. - :type parallel: - bool - :param flags: - Additional options to be forwarded to the scheduler. - :type flags: - list - :param force: - Ignore all warnings or checks during submission, just submit. - :type force: - bool - :param template: - The name of the template file to be used to generate the submission script. - :type template: - str - :param pretend: + _id : str + The _id to be used for this submission. (Default value = None) + parallel : bool + Execute all bundled operations in parallel. (Default value = False) + flags : list + Additional options to be forwarded to the scheduler. (Default value = None) + force : bool + Ignore all warnings or checks during submission, just submit. (Default value = False) + template : str + The name of the template file to be used to generate the submission + script. (Default value = "script.sh") + pretend : bool Do not actually submit, but only print the submission script to screen. Useful - for testing the submission workflow. - :type pretend: - bool - :param show_template_help: - Show information about available template variables and filters and exit. - :type show_template_help: - bool - :param \*\*kwargs: - Additional keyword arguments to be forwarded to the scheduler. - :return: + for testing the submission workflow. (Default value = False) + show_template_help : bool + Show information about available template variables and filters and + exit. (Default value = False) + env : :class:`~.ComputeEnvironment` + The environment to use for submission. Uses the environment defined + by the :class:`~.FlowProject` if None (Default value = None). + \*\*kwargs : + Additional keyword aruguments forwarded to :meth:`~.ComputeEnvironment.submit`. + + Returns + ------- + :class:`~.JobStatus` or None Returns the submission status after successful submission or None. + """ return self._submit_operations( operations, @@ -3748,44 +3772,39 @@ def submit( ignore_conditions_on_execution=IgnoreConditions.NONE, **kwargs, ): - """Submit function for the project's main submit interface. + r"""Submit function for the project's main submit interface. - :param bundle_size: + Parameters + ---------- + bundle_size : int Specify the number of operations to be bundled into one submission, defaults to 1. - :type bundle_size: - int - :param jobs: + jobs : Sequence of instances :class:`~signac.contrib.job.Job`. Only submit operations associated with the provided jobs. Defaults to all jobs. - :type jobs: - Sequence of instances :class:`~signac.contrib.job.Job`. - :param names: + names : Sequence of :class:`str` Only submit operations with any of the given names, defaults to all names. - :type names: - Sequence of :class:`str` - :param num: + num : int Limit the total number of submitted operations, defaults to no limit. - :type num: - int - :param parallel: - Execute all bundled operations in parallel. - :type parallel: - bool - :param force: - Ignore all warnings or checks during submission, just submit. - :type force: - bool - :param walltime: - Specify the walltime in hours or as instance of :class:`datetime.timedelta`. - :param ignore_conditions: - Specify if pre and/or post conditions check is to be ignored for eligibility check. - The default is :class:`IgnoreConditions.NONE`. - :type ignore_conditions: - :class:`~.IgnoreConditions` - :param ignore_conditions_on_execution: - Specify if pre and/or post conditions check is to be ignored for eligibility check after - submitting. The default is :class:`IgnoreConditions.NONE`. - :type ignore_conditions: - :class:`~.IgnoreConditions` + parallel : bool + Execute all bundled operations in parallel. (Default value = False) + force : bool + Ignore all warnings or checks during submission, just submit. (Default value = False) + walltime : + Specify the walltime in hours or as instance of + :class:`datetime.timedelta`. (Default value = None) + ignore_conditions : :class:`~.IgnoreConditions` + Specify if preconditions and/or postconditions are to be ignored + when determining eligibility. The default is + :class:`IgnoreConditions.NONE`. + ignore_conditions_on_execution : + Specify if preconditions and/or postconditions are to be ignored + when determining eligibility after submitting. The default is + :class:`IgnoreConditions.NONE`. + env : :class:`~.ComputeEnvironment` + The environment to use for submission. Uses the environment defined + by the :class:`~.FlowProject` if None (Default value = None). + \*\*kwargs : + Additional keyword aruguments forwarded to :meth:`~.ComputeEnvironment.submit`. + """ aggregates = self._convert_aggregates_from_jobs(jobs) @@ -3852,7 +3871,7 @@ def submit( @classmethod def _add_submit_args(cls, parser): - """Add arguments to submit sub command to parser.""" + """Add arguments to submit subcommand to parser.""" parser.add_argument( "flags", type=str, nargs="*", help="Flags to be forwarded to the scheduler." ) @@ -4012,7 +4031,7 @@ def _add_operation_bundling_arg_group(cls, parser): ) def export_job_statuses(self, collection, statuses): - """Export the job statuses to a database collection.""" + """Export the job statuses to a :class:`signac.Collection`.""" for status in statuses: job = self.open_job(id=status["job_id"]) status["statepoint"] = job.statepoint() @@ -4129,7 +4148,18 @@ def _add_print_status_args(cls, parser): def labels(self, job): """Yield all labels for the given ``job``. - See also: :meth:`~.label` + See also: :meth:`~.label`. + + Parameters + ---------- + job : :class:`signac.contrib.job.Job` + Job handle. + + Yields + ------ + str + Label value. + """ for label_func, label_name in self._label_functions.items(): if label_name is None: @@ -4154,25 +4184,25 @@ def labels(self, job): yield label_name def add_operation(self, name, cmd, pre=None, post=None, **kwargs): - """Add an operation to the workflow. + r"""Add an operation to the workflow. This method will add an instance of :class:`~.FlowOperation` to the operations of this project. .. seealso:: - A Python function may be defined as an operation function directly using - the :meth:`~.operation` decorator. + A Python function may be defined as an operation function directly + using the :meth:`~.operation` decorator. - Any FlowOperation is associated with a specific command, which should be - a function of :class:`~signac.contrib.job.Job`. The command (cmd) can - be stated as function, either by using str-substitution based on a job's - attributes, or by providing a unary callable, which expects an instance - of job as its first and only positional argument. + Any FlowOperation is associated with a specific command, which should + be a function of :class:`~signac.contrib.job.Job`. The command (cmd) + can be stated as function, either by using str-substitution based on a + job's attributes, or by providing a unary callable, which expects an + instance of job as its first and only positional argument. - For example, if we wanted to define a command for a program called 'hello', - which expects a job id as its first argument, we could construct the following - two equivalent operations: + For example, if we wanted to define a command for a program called + 'hello', which expects a job id as its first argument, we could + construct the following two equivalent operations: .. code-block:: python @@ -4186,36 +4216,34 @@ def add_operation(self, name, cmd, pre=None, post=None, **kwargs): # Substitute job state point parameters: op = FlowOperation('hello', cmd='cd {job.ws}; hello {job.sp.a}') - Pre-conditions (pre) and post-conditions (post) can be used to - trigger an operation only when certain conditions are met. Conditions are unary - callables, which expect an instance of job as their first and only positional - argument and return either True or False. + Preconditions (pre) and postconditions (post) can be used to trigger an + operation only when certain conditions are met. Conditions are unary + callables, which expect an instance of job as their first and only + positional argument and return either True or False. - An operation is considered "eligible" for execution when all pre-conditions - are met and when at least one of the post-conditions is not met. - Pre-conditions are always met when the list of pre-conditions is empty. - Post-conditions are never met when the list of post-conditions is empty. + An operation is considered "eligible" for execution when all + preconditions are met and when at least one of the postconditions is + not met. Preconditions are always met when the list of preconditions + is empty. Postconditions are never met when the list of postconditions + is empty. - Please note, eligibility in this contexts refers only to the workflow pipeline - and not to other contributing factors, such as whether the job-operation is currently - running or queued. + Please note, eligibility in this contexts refers only to the workflow + pipeline and not to other contributing factors, such as whether the + job-operation is currently running or queued. - :param name: + Parameters + ---------- + name : str A unique identifier for this operation, which may be freely chosen. - :type name: - str - :param cmd: + cmd : str or callable The command to execute operation; should be a function of job. - :type cmd: - str or callable - :param pre: - List of pre-conditions. - :type pre: - sequence of callables - :param post: - List of post-conditions. - :type post: - sequence of callables + pre : sequence of callables + List of preconditions. (Default value = None) + post : sequence of callables + List of postconditions. (Default value = None) + \*\*kwargs : + Keyword arguments passed as directives. + """ if name in self.operations: raise KeyError("An operation with this identifier is already added.") @@ -4229,14 +4257,16 @@ def add_operation(self, name, cmd, pre=None, post=None, **kwargs): def completed_operations(self, job): """Determine which operations have been completed for job. - :param job: + Parameters + ---------- + job : :class:`~signac.contrib.job.Job` The signac job handle. - :type job: - :class:`~signac.contrib.job.Job` - :return: - The name of the operations that are complete. - :rtype: - str + + Yields + ------ + str + The names of the operations that are complete. + """ for name, op in self._operations.items(): if op._complete((job,)): @@ -4245,18 +4275,21 @@ def completed_operations(self, job): def _next_operations(self, jobs=None, ignore_conditions=IgnoreConditions.NONE): """Determine the next eligible operations for aggregates. - :param jobs: - The signac job handles. By default all the aggregates are evaluated to get - the next operation associated. - :type jobs: - tuple of :class:`~signac.contrib.job.Job` - :param ignore_conditions: - Specify if pre and/or post conditions check is to be ignored for eligibility check. - The default is :class:`IgnoreConditions.NONE`. - :type ignore_conditions: - :class:`~.IgnoreConditions` - :yield: - All instances of :class:`~._JobOperation` jobs are eligible for. + Parameters + ---------- + jobs : tuple of :class:`~signac.contrib.job.Job` + The signac job handles. By default all the aggregates are evaluated + to get the next operation associated. + ignore_conditions : :class:`~.IgnoreConditions` + Specify if preconditions and/or postconditions are to be ignored + when determining eligibility. The default is + :class:`IgnoreConditions.NONE`. + + Yields + ------ + :class:`~._JobOperation` + All eligible operations for the provided jobs. + """ for name in self.operations: group = self._groups[name] @@ -4274,17 +4307,20 @@ def _next_operations(self, jobs=None, ignore_conditions=IgnoreConditions.NONE): def next_operations(self, *jobs, ignore_conditions=IgnoreConditions.NONE): r"""Determine the next eligible operations for provided job(s). - :param \*jobs: - The signac job handles. - :type \*jobs: - One or more instances of :class:`~signac.contrib.job.Job`. - :param ignore_conditions: - Specify if pre and/or post conditions check is to be ignored for eligibility check. - The default is :class:`IgnoreConditions.NONE`. - :type ignore_conditions: - :class:`~.IgnoreConditions` - :yield: - All instances of :class:`~.JobOperation` jobs are eligible for. + Parameters + ---------- + \*jobs : One or more instances of :class:`.Job`. + Jobs. + ignore_conditions : :class:`~.IgnoreConditions` + Specify if preconditions and/or postconditions are to be ignored + while checking eligibility. The default is + :class:`IgnoreConditions.NONE`. + + Yields + ------ + :class:`~.JobOperation` : + Eligible job operation. + """ for name in self.operations: group = self._groups[name] @@ -4318,9 +4354,9 @@ def next_operations(self, *jobs, ignore_conditions=IgnoreConditions.NONE): @classmethod def operation(cls, func, name=None): - """Add the function `func` as operation function to the class workflow definition. + """Add an operation function to the class workflow definition. - This function is designed to be used as a decorator function, for example: + This function is designed to be used as a decorator, for example: .. code-block:: python @@ -4329,6 +4365,20 @@ def hello(job): print('Hello', job) See also: :meth:`~.flow.FlowProject.add_operation`. + + Parameters + ---------- + func : callable + The function to add to the workflow. + name : str + The operation name. Uses the name of the function if None. + (Default value = None) + + Returns + ------- + callable + The operation function. + """ if isinstance(func, str): return lambda op: cls.operation(op, name=func) @@ -4381,12 +4431,12 @@ def _collect_conditions(cls, attr): @classmethod def _collect_pre_conditions(cls): - """Collect all pre-conditions added with the ``@FlowProject.pre`` decorator.""" + """Collect all preconditions added with the ``@FlowProject.pre`` decorator.""" return cls._collect_conditions("_OPERATION_PRE_CONDITIONS") @classmethod def _collect_post_conditions(cls): - """Collect all post-conditions added with the ``@FlowProject.post`` decorator.""" + """Collect all postconditions added with the ``@FlowProject.post`` decorator.""" return cls._collect_conditions("_OPERATION_POST_CONDITIONS") def _register_operations(self): @@ -4399,7 +4449,7 @@ def _register_operations(self): if name in self._operations: raise ValueError(f"Repeat definition of operation with name '{name}'.") - # Extract pre/post conditions and directives from function: + # Extract preconditions/postconditions and directives from function: params = { "pre": pre_conditions.get(func, None), "post": post_conditions.get(func, None), @@ -4422,6 +4472,13 @@ def _register_aggregates(self): def make_group(cls, name, options=""): """Make a FlowGroup named ``name`` and return a decorator to make groups. + FlowGroups group operations together for running and submitting + JobOperations. + + Examples + -------- + The code below creates a group and adds an operation to that group. + .. code-block:: python example_group = FlowProject.make_group('example') @@ -4431,21 +4488,19 @@ def make_group(cls, name, options=""): def foo(job): return "hello world" - FlowGroups group operations together for running and submitting - JobOperations. - - :param name: + Parameters + ---------- + name : str The name of the :class:`~.FlowGroup`. - :type name: - str - :param options: - A string to append to submissions can be any valid :meth:`FlowOperation.run` option. - :type options: - str - :param aggregator_obj: - aggregator object associated with the :class:`FlowGroup` - :type aggregator_obj: - :class:`aggregator` + options : str + A string to append to submissions. Can be any valid + :meth:`FlowOperation.run` option. (Default value = "") + + Returns + ------- + :class:`~.FlowGroupEntry` + The created group. + """ if name in cls._GROUP_NAMES: raise ValueError(f"Repeat definition of group with name '{name}'.") @@ -4504,14 +4559,16 @@ def groups(self): def _get_aggregate_store(self, group): """Return aggregate store associated with the FlowGroup. - :param group: + Parameters + ---------- + group : str The name of the FlowGroup whose aggregate store will be returned. - :type group: - str - :returns: + + Returns + ------- + :class:`_DefaultAggregateStore` Aggregate store containing aggregates associated with the provided FlowGroup. - :rtype: - :class:`_DefaultAggregateStore` + """ for aggregate_store, groups in self._stored_aggregates.items(): if group in groups: @@ -4755,6 +4812,13 @@ class MyProject(FlowProject): .. code-block:: bash $ python my_project.py --help + + Parameters + ---------- + parser : :class:`argparse.ArgumentParser` + The argument parser used to implement the command line interface. + If None, a new parser is constructed. (Default value = None) + """ # Find file that main is called in. When running through the command # line interface, we know exactly what the entrypoint path should be: @@ -4847,7 +4911,7 @@ class MyProject(FlowProject): default=1, help="Specify how many times a particular job-operation may be executed within one " "session (default=1). This is to prevent accidental infinite loops, " - "where operations are executed indefinitely, because post conditions " + "where operations are executed indefinitely, because postconditions " "were not properly set. Use -1 to allow for an infinite number of passes.", ) execution_group.add_argument( From 0701888e4c8a245e71d837b72cc238a583067e63 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 8 Dec 2020 21:16:35 -0600 Subject: [PATCH 18/34] Fix remaining sphinx-style docstrings. --- flow/aggregates.py | 12 +++++++++--- flow/environment.py | 24 +++++++++++++++--------- flow/environments/incite.py | 4 ++-- flow/environments/xsede.py | 7 ++++--- flow/labels.py | 36 +++++++++++++++++++++++++++--------- flow/operations.py | 12 +++++++++--- flow/project.py | 12 +++++++++--- 7 files changed, 75 insertions(+), 32 deletions(-) diff --git a/flow/aggregates.py b/flow/aggregates.py index ed9b97cc2..37df9fa57 100644 --- a/flow/aggregates.py +++ b/flow/aggregates.py @@ -372,10 +372,16 @@ def __getitem__(self, id): def __contains__(self, id): """Return whether this instance contains an aggregate (by aggregate id). - :param id: + Parameters + ---------- + id : str The id of an aggregate of jobs. - :type id: - str + + Returns + ------- + bool + Whether this instance contains an aggregate. + """ return id in self._aggregate_per_id diff --git a/flow/environment.py b/flow/environment.py index 6ce2dbe08..a50a9dd05 100644 --- a/flow/environment.py +++ b/flow/environment.py @@ -109,7 +109,8 @@ def template_filter(func): Returns ------- - callable : Decorated function. + callable + Decorated function. """ setattr(func, "_flow_template_filter", True) @@ -184,7 +185,8 @@ def submit(cls, script, flags=None, *args, **kwargs): Returns ------- - JobStatus.submitted or None : Status of job, if submitted. + JobStatus.submitted or None + Status of job, if submitted. """ if flags is None: @@ -238,7 +240,7 @@ def get_config_value(cls, key, default=flow_config._GET_CONFIG_VALUE_NONE): Returns ------- - type + object The value or default value. Raises @@ -261,7 +263,8 @@ def _get_omp_prefix(cls, operation): Returns ------- - str : The prefix to be added to the operation's command. + str + The prefix to be added to the operation's command. """ return "export OMP_NUM_THREADS={}; ".format( @@ -280,11 +283,11 @@ def _get_mpi_prefix(cls, operation, parallel): If True, operations are assumed to be executed in parallel, which means that the number of total tasks is the sum of all tasks instead of the maximum number of tasks. Default is set to False. - :return mpi_prefix: The prefix should be added for the operation. Returns ------- - str : The prefix to be added to the operation's command. + str + The prefix to be added to the operation's command. """ if operation.directives.get("nranks"): @@ -312,7 +315,8 @@ def get_prefix(cls, operation, parallel=False, mpi_prefix=None, cmd_prefix=None) Returns ------- - str : The prefix to be added to the operation's command. + str + The prefix to be added to the operation's command. """ prefix = "" @@ -544,7 +548,8 @@ def registered_environments(import_configured=True): Returns ------- - list : List of registered environments. + list + List of registered environments. """ if import_configured: @@ -570,7 +575,8 @@ def get_environment(test=False, import_configured=True): Returns ------- - :class:`~.ComputeEnvironment` : The detected environment class. + :class:`~.ComputeEnvironment` + The detected environment class. """ if test: diff --git a/flow/environments/incite.py b/flow/environments/incite.py index f19549089..71eec3a75 100644 --- a/flow/environments/incite.py +++ b/flow/environments/incite.py @@ -158,11 +158,11 @@ def _get_mpi_prefix(cls, operation, parallel): If True, operations are assumed to be executed in parallel, which means that the number of total tasks is the sum of all tasks instead of the maximum number of tasks. Default is set to False. - :return mpi_prefix: The prefix should be added for the operation. Returns ------- - str : The prefix to be added to the operation's command. + str + The prefix to be added to the operation's command. """ extra_args = str(operation.directives.get("extra_jsrun_args", "")) diff --git a/flow/environments/xsede.py b/flow/environments/xsede.py index 56189db75..9cde1b126 100644 --- a/flow/environments/xsede.py +++ b/flow/environments/xsede.py @@ -82,7 +82,8 @@ def return_and_increment(cls, increment): Returns ------- - int : The value of the base offset before incrementing. + int + The value of the base offset before incrementing. """ cls.base_offset += increment @@ -157,11 +158,11 @@ def _get_mpi_prefix(cls, operation, parallel): If True, operations are assumed to be executed in parallel, which means that the number of total tasks is the sum of all tasks instead of the maximum number of tasks. Default is set to False. - :return mpi_prefix: The prefix should be added for the operation. Returns ------- - str : The prefix to be added to the operation's command. + str + The prefix to be added to the operation's command. """ if operation.directives.get("nranks"): diff --git a/flow/labels.py b/flow/labels.py index 1f250cee9..54a5e5dff 100644 --- a/flow/labels.py +++ b/flow/labels.py @@ -31,10 +31,16 @@ def __call__(self, func): This call operator allows the class to be used as a decorator. - :param func: + Parameters + ---------- + func : callable The function to decorate. - :type func: - callable + + Returns + ------- + callable + The decorated function. + """ func._label = True if self.name is not None: @@ -53,10 +59,16 @@ def __call__(self, func): This call operator allows the class to be used as a decorator. - :param func: + Parameters + ---------- + func : callable The function to decorate. - :type func: - callable + + Returns + ------- + callable + The decorated function. + """ return staticmethod(super().__call__(func)) @@ -72,10 +84,16 @@ def __call__(self, func): This call operator allows the class to be used as a decorator. - :param func: + Parameters + ---------- + func : callable The function to decorate. - :type func: - callable + + Returns + ------- + callable + The decorated function. + """ return classmethod(super().__call__(func)) diff --git a/flow/operations.py b/flow/operations.py index 0a871df43..24ab86c38 100644 --- a/flow/operations.py +++ b/flow/operations.py @@ -138,10 +138,16 @@ def __call__(self, func): This call operator allows the class to be used as a decorator. - :param func: + Parameters + ---------- + func : callable The function to decorate. - :type func: - callable + + Returns + ------- + callable + The decorated function. + """ directives = getattr(func, "_flow_directives", {}) directives.update(self.kwargs) diff --git a/flow/project.py b/flow/project.py index 9bc080946..bad449475 100644 --- a/flow/project.py +++ b/flow/project.py @@ -781,10 +781,16 @@ def __call__(self, func): This call operator allows the class to be used as a decorator. - :param func: + Parameters + ---------- + func : callable The function to decorate. - :type func: - callable + + Returns + ------- + callable + The decorated function. + """ if hasattr(func, "_flow_groups"): if self.name in func._flow_groups: From 9a19fac67e9a32ecd148931723b866808a3ac332 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 8 Dec 2020 22:08:08 -0600 Subject: [PATCH 19/34] Updated docstrings with velin (https://github.com/Carreau/velin). --- flow/aggregates.py | 10 ++++------ flow/directives.py | 4 ++-- flow/environment.py | 8 ++++---- flow/operations.py | 1 - flow/project.py | 19 +++++++++---------- flow/template.py | 25 ++++++++++++++----------- flow/testing.py | 8 ++++---- flow/util/config.py | 2 +- flow/util/misc.py | 2 +- 9 files changed, 39 insertions(+), 40 deletions(-) diff --git a/flow/aggregates.py b/flow/aggregates.py index 37df9fa57..8f6246582 100644 --- a/flow/aggregates.py +++ b/flow/aggregates.py @@ -88,9 +88,8 @@ def groupsof(cls, num=1, sort_by=None, sort_ascending=True, select=None): project and they are aggregated in groups of 3, then the generated aggregates will have lengths 3, 3, 3, and 1. - Example - ------- - + Examples + -------- The code block below shows how to aggregate jobs in groups of 2. .. code-block:: python @@ -147,9 +146,8 @@ def aggregator_function(jobs): def groupby(cls, key, default=None, sort_by=None, sort_ascending=True, select=None): """Aggregate jobs according to matching state point values. - Example - ------- - + Examples + -------- The code block below provides an example of how to aggregate jobs having a common state point parameter ``'sp'`` whose value, if not found, is replaced by a default value of -1. diff --git a/flow/directives.py b/flow/directives.py index 0493e98ef..e832c1819 100644 --- a/flow/directives.py +++ b/flow/directives.py @@ -196,7 +196,7 @@ def update(self, other, aggregate=False, jobs=None, parallel=False): The other set of directives. aggregate : bool Whether to combine directives according to serial/parallel rules. - jobs : + jobs The jobs used to evaluate directives. parallel : bool Whether to aggregate according to parallel rules. @@ -215,7 +215,7 @@ def evaluate(self, jobs): Parameters ---------- - jobs : + jobs The jobs used to evaluate directives. """ diff --git a/flow/environment.py b/flow/environment.py index a50a9dd05..b6c8430d2 100644 --- a/flow/environment.py +++ b/flow/environment.py @@ -543,8 +543,8 @@ def registered_environments(import_configured=True): Parameters ---------- import_configured : bool - Whether to import environments specified in the flow configuration. - (Default value = True) + Whether to import environments specified in the flow configuration. + (Default value = True) Returns ------- @@ -570,8 +570,8 @@ def get_environment(test=False, import_configured=True): test : bool Whether to return the TestEnvironment. (Default value = False) import_configured : bool - Whether to import environments specified in the flow configuration. - (Default value = True) + Whether to import environments specified in the flow configuration. + (Default value = True) Returns ------- diff --git a/flow/operations.py b/flow/operations.py index 24ab86c38..51c925121 100644 --- a/flow/operations.py +++ b/flow/operations.py @@ -203,7 +203,6 @@ def hello(job): You can control the degree of parallelization with the ``--np`` argument. - For more information, see: .. code-block:: bash diff --git a/flow/project.py b/flow/project.py index bad449475..7daaaab26 100644 --- a/flow/project.py +++ b/flow/project.py @@ -239,7 +239,6 @@ def _make_bundles(operations, size=None): ---------- operations : iterable Iterable of operations. - size : int Size of bundles. (Default value = None) @@ -2581,8 +2580,8 @@ def print_status( output_format : str Status output format, supports: 'terminal' (default), 'markdown' or 'html'. - no_parallelize : - (Default value = False) + no_parallelize : bool + Disable parallelization. (Default value = False) Returns ------- @@ -3162,7 +3161,7 @@ def run( Parameters ---------- - jobs : iterable of :class:`~signac.contrib.job.Job` or aggregates of jobs. + jobs : iterable of :class:`~signac.contrib.job.Job` or aggregates of jobs Only execute operations for the given jobs or aggregates of jobs, or all if the argument is omitted. (Default value = None) names : iterable of :class:`str` @@ -3630,7 +3629,7 @@ def _submit_operations( env : :class:`~.ComputeEnvironment` The environment to use for submission. Uses the environment defined by the :class:`~.FlowProject` if None (Default value = None). - \*\*kwargs : + \*\*kwargs Additional keyword aruguments forwarded to :meth:`~.ComputeEnvironment.submit`. Returns @@ -3742,7 +3741,7 @@ def submit_operations( env : :class:`~.ComputeEnvironment` The environment to use for submission. Uses the environment defined by the :class:`~.FlowProject` if None (Default value = None). - \*\*kwargs : + \*\*kwargs Additional keyword aruguments forwarded to :meth:`~.ComputeEnvironment.submit`. Returns @@ -3794,21 +3793,21 @@ def submit( Execute all bundled operations in parallel. (Default value = False) force : bool Ignore all warnings or checks during submission, just submit. (Default value = False) - walltime : + walltime : :class:`datetime.timedelta` Specify the walltime in hours or as instance of :class:`datetime.timedelta`. (Default value = None) ignore_conditions : :class:`~.IgnoreConditions` Specify if preconditions and/or postconditions are to be ignored when determining eligibility. The default is :class:`IgnoreConditions.NONE`. - ignore_conditions_on_execution : + ignore_conditions_on_execution : :class:`~.IgnoreConditions` Specify if preconditions and/or postconditions are to be ignored when determining eligibility after submitting. The default is :class:`IgnoreConditions.NONE`. env : :class:`~.ComputeEnvironment` The environment to use for submission. Uses the environment defined by the :class:`~.FlowProject` if None (Default value = None). - \*\*kwargs : + \*\*kwargs Additional keyword aruguments forwarded to :meth:`~.ComputeEnvironment.submit`. """ @@ -4247,7 +4246,7 @@ def add_operation(self, name, cmd, pre=None, post=None, **kwargs): List of preconditions. (Default value = None) post : sequence of callables List of postconditions. (Default value = None) - \*\*kwargs : + \*\*kwargs Keyword arguments passed as directives. """ diff --git a/flow/template.py b/flow/template.py index b0333bfbd..6321a3fe8 100644 --- a/flow/template.py +++ b/flow/template.py @@ -35,22 +35,25 @@ def init(alias=None, template=None, root=None, out=None): Parameters ---------- alias : str - Python identifier used as a file name for the template output. Uses - ``"project"`` if None. (Default value = None) + Python identifier used as a file name for the template output. Uses + ``"project"`` if None. (Default value = None) + template : str - Name of the template to use. Built-in templates are: + Name of the template to use. Built-in templates are: + + * ``"minimal"`` + * ``"example"`` + * ``"testing"`` - * ``"minimal"`` - * ``"example"`` - * ``"testing"`` + Uses ``"minimal"`` if None. (Default value = None) - Uses ``"minimal"`` if None. (Default value = None) root : str - Directory where the output file is placed. Uses the current working - directory if None. (Default value = None) + Directory where the output file is placed. Uses the current working + directory if None. (Default value = None) + out : file-like object - The stream where output is printed. Uses :obj:`sys.stderr` if None. - (Default value = None) + The stream where output is printed. Uses :obj:`sys.stderr` if None. + (Default value = None) Returns ------- diff --git a/flow/testing.py b/flow/testing.py index b373f760d..3c3c9d872 100644 --- a/flow/testing.py +++ b/flow/testing.py @@ -16,11 +16,11 @@ def make_project(alias="project", root=None, **kwargs): Parameters ---------- alias : str - Python identifier used as a file name for the template output. - (Default value = ``"project"``) + Python identifier used as a file name for the template output. + (Default value = ``"project"``) root : str - Directory where the output file is placed. Uses the current working - directory if None. (Default value = None) + Directory where the output file is placed. Uses the current working + directory if None. (Default value = None) \*\*kwargs Keyword arguments forwarded to :meth:`signac.testing.init_jobs`. diff --git a/flow/util/config.py b/flow/util/config.py index 1e5b00d54..f3bbc2226 100644 --- a/flow/util/config.py +++ b/flow/util/config.py @@ -34,7 +34,7 @@ def require_config_value(key, ns=None, default=_GET_CONFIG_VALUE_NONE): The environment specific configuration key. ns : str The namespace in which to look for the key. (Default value = None) - default : + default A default value in case the key cannot be found within the user's configuration. diff --git a/flow/util/misc.py b/flow/util/misc.py index 27e39e1ba..46581100d 100644 --- a/flow/util/misc.py +++ b/flow/util/misc.py @@ -218,7 +218,7 @@ def _to_hashable(obj): Parameters ---------- - obj : + obj Object to make hashable. Lists are converted to tuples, and hashes are defined for dicts. From 1b783abd096d0822de9a2be611ad0b31c2ba2d12 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 8 Dec 2020 22:16:06 -0600 Subject: [PATCH 20/34] Adopt numpy convention for pydocstyle. --- doc/conf.py | 1 + flow/aggregates.py | 2 ++ flow/project.py | 1 + flow/scheduling/base.py | 1 + flow/util/misc.py | 1 + setup.cfg | 2 +- 6 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index f22e28146..baac9f556 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -20,6 +20,7 @@ class Mock(MagicMock): @classmethod def __getattr__(cls, name): + """Get mocked attribute.""" if name == "_mock_methods": return [] return Mock() diff --git a/flow/aggregates.py b/flow/aggregates.py index 8f6246582..6d7ec488f 100644 --- a/flow/aggregates.py +++ b/flow/aggregates.py @@ -240,6 +240,7 @@ def aggregator_function(jobs): return cls(aggregator_function, sort_by, sort_ascending, select) def __eq__(self, other): + """Test equality with another aggregator.""" return ( type(self) == type(other) and not self._is_aggregate @@ -247,6 +248,7 @@ def __eq__(self, other): ) def __hash__(self): + """Hash this aggregator.""" return hash( ( self._sort_by, diff --git a/flow/project.py b/flow/project.py index 7daaaab26..590062992 100644 --- a/flow/project.py +++ b/flow/project.py @@ -726,6 +726,7 @@ def __init__(self, op_func, pre=None, post=None): self._op_func = op_func def __str__(self): + """Return string representing operation.""" return f"{type(self).__name__}(op_func='{self._op_func}')" def __call__(self, *jobs): diff --git a/flow/scheduling/base.py b/flow/scheduling/base.py index e77ed83c3..4ae77fd0b 100644 --- a/flow/scheduling/base.py +++ b/flow/scheduling/base.py @@ -58,6 +58,7 @@ def _id(self): return self._job_id def __str__(self): + """Return job ID string.""" return str(self._id()) def name(self): diff --git a/flow/util/misc.py b/flow/util/misc.py index 46581100d..3d89a428c 100644 --- a/flow/util/misc.py +++ b/flow/util/misc.py @@ -169,6 +169,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) def __getitem__(self, key): + """Get item by key.""" self._keys_used.add(key) return super().__getitem__(key) diff --git a/setup.cfg b/setup.cfg index e016730ed..3d1a4f14f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,10 +11,10 @@ description-file = README.md python-tag = py3 [pydocstyle] +convention = numpy match = ^((?!\.sync-zenodo-metadata|setup).)*\.py$ match-dir = ^((?!\.|tests|mistune).)*$ ignore-decorators = "deprecated" -ignore = D105,D107,D203,D204,D213 [flake8] max-line-length = 100 From 88d64a0694d58e2a6dbfd3d1d1cc378c320685c6 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 8 Dec 2020 22:16:25 -0600 Subject: [PATCH 21/34] Remove methods that are implemented identically in the parent class. --- flow/scheduling/torque.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/flow/scheduling/torque.py b/flow/scheduling/torque.py index d32ac9e67..d49bdab21 100644 --- a/flow/scheduling/torque.py +++ b/flow/scheduling/torque.py @@ -68,13 +68,6 @@ def __init__(self, node): def _id(self): return self.node.find("Job_Id").text - def __str__(self): - return str(self._id()) - - def name(self): - """Return the name of the cluster job.""" - return self.node.find("Job_Name").text - def status(self): """Return the status of the cluster job.""" job_state = self.node.find("job_state").text From 398770ab07cd2226d8fba75cb407c3884727ebc5 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 9 Dec 2020 00:36:45 -0600 Subject: [PATCH 22/34] Add call operators to docs. --- doc/api.rst | 23 +++++++++++++++++++++++ flow/project.py | 3 ++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/doc/api.rst b/doc/api.rst index ded5051ca..9077b539a 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -109,10 +109,12 @@ Operations and Status .. autoclass:: flow.project.FlowOperation :show-inheritance: :members: + :special-members: __call__ .. autoclass:: flow.project.FlowCmdOperation :show-inheritance: :members: + :special-members: __call__ .. autoclass:: flow.project.JobOperation :members: @@ -120,6 +122,20 @@ Operations and Status .. autoclass:: flow.render_status.Renderer :members: generate_terminal_output, generate_html_output, render +Labels +------ + +.. autoclass:: flow.label + :members: + :special-members: __call__ + +.. autoclass:: flow.staticlabel + :members: + :special-members: __call__ + +.. autoclass:: flow.classlabel + :members: + :special-members: __call__ @flow.cmd --------- @@ -135,6 +151,8 @@ Operations and Status ---------------- .. autoclass:: directives + :members: + :special-members: __call__ flow.run() ---------- @@ -157,6 +175,10 @@ The FlowGroup .. autoclass:: flow.project.FlowGroup :members: +.. autoclass:: flow.project.FlowGroupEntry + :members: + :special-members: __call__ + Aggregation ----------- @@ -164,6 +186,7 @@ Aggregation .. autoclass:: flow.aggregates.aggregator :members: + :special-members: __call__ .. autofunction:: flow.aggregates.get_aggregate_id diff --git a/flow/project.py b/flow/project.py index 590062992..17e833f40 100644 --- a/flow/project.py +++ b/flow/project.py @@ -682,6 +682,7 @@ def __str__(self): return f"{type(self).__name__}(cmd='{self._cmd}')" def __call__(self, *jobs, **kwargs): + """Return the command formatted with the supplied job(s).""" job = kwargs.pop("job", None) if kwargs: raise ValueError(f"Invalid keyword arguments: {', '.join(kwargs)}") @@ -734,7 +735,7 @@ def __call__(self, *jobs): Parameters ---------- - \*jobs : One or more instances of :class:`.Job`. + \*jobs : One or more instances of :class:`~signac.contrib.job.Job`. The jobs passed to the operation. Returns From e4a76db22803d306b09e5af6d048e11aed098bcc Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 9 Dec 2020 14:56:22 -0600 Subject: [PATCH 23/34] Fix FlowGroup reference. --- flow/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flow/project.py b/flow/project.py index 17e833f40..f04cd3de1 100644 --- a/flow/project.py +++ b/flow/project.py @@ -377,7 +377,7 @@ def get_status(self): class JobOperation(_JobOperation): """Class containing execution information for one group and one job. - The execution or submission of a :class:`FlowGroup` uses a passed-in command + The execution or submission of a :class:`~.FlowGroup` uses a passed-in command which can either be a string or function with no arguments that returns a shell executable command. The shell executable command won't be used if it is determined that the group can be executed without forking. From 24cdf75c511141a76dc309998e09b9e03bb51ed9 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 9 Dec 2020 23:49:42 -0600 Subject: [PATCH 24/34] Reformat examples. --- flow/aggregates.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/flow/aggregates.py b/flow/aggregates.py index 6d7ec488f..d7e498daa 100644 --- a/flow/aggregates.py +++ b/flow/aggregates.py @@ -18,9 +18,8 @@ class aggregator: By default, if the ``aggregator_function`` is ``None``, an aggregate of all jobs will be created. - Example - ------- - + Examples + -------- The code block below defines a :class:`~.FlowOperation` that prints the total length of the provided aggregate of jobs. From 09160d88a1aec5aafdef2c6246a0b8c968601480 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 9 Dec 2020 23:51:47 -0600 Subject: [PATCH 25/34] Remove trailing colons where no type is specified. --- flow/aggregates.py | 6 +++--- flow/environments/incite.py | 2 +- flow/project.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/flow/aggregates.py b/flow/aggregates.py index d7e498daa..b81fca8fe 100644 --- a/flow/aggregates.py +++ b/flow/aggregates.py @@ -276,7 +276,7 @@ def _get_unique_function_id(self, func): Returns ------- - str : + str The hash of the function's bytecode if possible, otherwise the hash of the function. @@ -302,7 +302,7 @@ def _create_AggregatesStore(self, project): Returns ------- - :class:`~._DefaultAggregateStore` or :class:`~._AggregatesStore` : + :class:`~._DefaultAggregateStore` or :class:`~._AggregatesStore` The aggregate store. """ @@ -549,7 +549,7 @@ def get_aggregate_id(jobs): Returns ------- - str : + str The generated aggregate id. """ diff --git a/flow/environments/incite.py b/flow/environments/incite.py index 71eec3a75..f99e36b69 100644 --- a/flow/environments/incite.py +++ b/flow/environments/incite.py @@ -129,7 +129,7 @@ def jsrun_options(cls, resource_set): Parameters ---------- - resource_set : + resource_set : tuple Tuple of *(Number of resource sets, tasks (MPI Ranks) per resource set, physical cores (CPUs) per resource set, GPUs per resource set)*. diff --git a/flow/project.py b/flow/project.py index f04cd3de1..ea152d373 100644 --- a/flow/project.py +++ b/flow/project.py @@ -471,7 +471,7 @@ class _SubmissionJobOperation(_JobOperation): first pass of :meth:`FlowProject.run` because all postconditions are met. These operations may be executed in subsequent iterations of the run loop. - \*\*kwargs : + \*\*kwargs Passed to the constructor of :class:`_JobOperation`. """ @@ -4325,7 +4325,7 @@ def next_operations(self, *jobs, ignore_conditions=IgnoreConditions.NONE): Yields ------ - :class:`~.JobOperation` : + :class:`~.JobOperation` Eligible job operation. """ From 4b7f721aca278caa649e99ef9e6fb670c57ae7c1 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 9 Dec 2020 23:53:54 -0600 Subject: [PATCH 26/34] Fix class reference. --- flow/environments/incite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flow/environments/incite.py b/flow/environments/incite.py index f99e36b69..3d5bbab86 100644 --- a/flow/environments/incite.py +++ b/flow/environments/incite.py @@ -84,7 +84,7 @@ def guess_resource_sets(cls, operation): Parameters ---------- - operation : class:`flow.BaseFlowOperation` + operation : :class:`flow.BaseFlowOperation` The operation whose directives will be used to compute the resource set. From 8d8646e5be1316802725a3231a0fb23f16203ce4 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 9 Dec 2020 23:55:14 -0600 Subject: [PATCH 27/34] Fix (deprecated) JobOperation references. --- flow/project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flow/project.py b/flow/project.py index ea152d373..248876e6c 100644 --- a/flow/project.py +++ b/flow/project.py @@ -3023,7 +3023,7 @@ def run_operations( Parameters ---------- - operations : Sequence of instances of :class:`.JobOperation` + operations : Sequence of instances of :class:`~.JobOperation` The operations to execute (optional). (Default value = None) pretend : bool Do not actually execute the operations, but show the commands that @@ -3721,7 +3721,7 @@ def submit_operations( Parameters ---------- - operations : A sequence of instances of :class:`.JobOperation` + operations : A sequence of instances of :class:`~.JobOperation` The operations to submit. _id : str The _id to be used for this submission. (Default value = None) From 9009988019f01447a71050d1839cab2ae91184e8 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 10 Dec 2020 00:00:27 -0600 Subject: [PATCH 28/34] Use preconditions/postconditions. --- tests/test_project.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/test_project.py b/tests/test_project.py index ac295118f..7c1d487c8 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -291,11 +291,11 @@ def label2(job): assert len(c._label_functions) == 1 def test_conditions_with_inheritance(self): - """Tests the inheritance of pre/post conditions. + """Tests the inheritance of preconditions/postconditions. - Class A should only have one pre/post condition, while class C that - inherits from A should have three, and class B should just have two - explicitly defined. Proper execution is tested in the + Class A should only have one precondition/postcondition, while class C + that inherits from A should have three, and class B should just have + two explicitly defined. Proper execution is tested in the TestExecutionProject. """ @@ -529,7 +529,7 @@ def op4(job): assert job.doc.b assert job.doc.c - def test_pre_post_condition(self): + def test_precondition_postcondition(self): class A(FlowProject): pass @@ -1080,8 +1080,8 @@ def op1(job): assert len(project) with redirect_stderr(StringIO()): for state, expected_evaluation in [ - (0b0000, 0b1000), # First pre-condition is not met - (0b0001, 0b1000), # means only the first pre-cond. + (0b0000, 0b1000), # First precondition is not met + (0b0001, 0b1000), # means only the first precondition (0b0010, 0b1000), # should be evaluated. (0b0011, 0b1000), (0b0100, 0b1000), @@ -1089,12 +1089,12 @@ def op1(job): (0b0110, 0b1000), (0b0111, 0b1000), (0b1000, 0b1100), # The first, but not the second - (0b1001, 0b1100), # pre-condition is met, need to evaluate - (0b1010, 0b1100), # both pre-conditions, but not post-conditions. + (0b1001, 0b1100), # precondition is met, need to evaluate + (0b1010, 0b1100), # both preconditions, but not postconditions. (0b1011, 0b1100), - (0b1100, 0b1110), # Both pre-conditions met, evaluate - (0b1101, 0b1110), # first post-condition. - (0b1110, 0b1111), # All pre-conditions and 1st post-condition + (0b1100, 0b1110), # Both preconditions met, evaluate + (0b1101, 0b1110), # first postcondition. + (0b1110, 0b1111), # All preconditions and 1st postcondition # are met, need to evaluate all. (0b1111, 0b1111), # All conditions met, need to evaluate all. ]: From c2f9195b32d5129909008bab97a798f80d8114de Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 10 Dec 2020 00:06:16 -0600 Subject: [PATCH 29/34] Add changelog line. --- changelog.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.txt b/changelog.txt index 931fb8980..eb6c31fe9 100644 --- a/changelog.txt +++ b/changelog.txt @@ -21,6 +21,7 @@ Changed +++++++ - Command line interface always uses ``--job-id`` instead of ``--jobid`` (#363, #386). - ``CPUEnvironment`` and ``GPUEnvironment`` classes are deprecated (#381). +- Docstrings are now written in `numpydoc style `__ (#392). Version 0.11 ============ From 3737c692ca1ee3964afc76b6af76ac2a445152d0 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 11 Dec 2020 11:16:26 -0600 Subject: [PATCH 30/34] Apply suggestions from code review Co-authored-by: Brandon Butler --- flow/directives.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flow/directives.py b/flow/directives.py index e832c1819..1898fc9a8 100644 --- a/flow/directives.py +++ b/flow/directives.py @@ -196,7 +196,7 @@ def update(self, other, aggregate=False, jobs=None, parallel=False): The other set of directives. aggregate : bool Whether to combine directives according to serial/parallel rules. - jobs + jobs : :class:`signac.contrib.job.Job` or tuple of :class:`signac.contrib.job.Job` The jobs used to evaluate directives. parallel : bool Whether to aggregate according to parallel rules. @@ -215,7 +215,7 @@ def evaluate(self, jobs): Parameters ---------- - jobs + jobs : :class:`signac.contrib.job.Job` or tuple of :class:`signac.contrib.job.Job` The jobs used to evaluate directives. """ From 5c506ce52fc979f32c2452277ee6ed5134cfb7b2 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 11 Dec 2020 11:33:52 -0600 Subject: [PATCH 31/34] Move env argument. --- flow/project.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flow/project.py b/flow/project.py index 248876e6c..3a37d4068 100644 --- a/flow/project.py +++ b/flow/project.py @@ -3613,6 +3613,9 @@ def _submit_operations( The operations to submit. _id : str The _id to be used for this submission. (Default value = None) + env : :class:`~.ComputeEnvironment` + The environment to use for submission. Uses the environment defined + by the :class:`~.FlowProject` if None (Default value = None). parallel : bool Execute all bundled operations in parallel. (Default value = False) flags : list @@ -3628,9 +3631,6 @@ def _submit_operations( show_template_help : bool Show information about available template variables and filters and exit. (Default value = False) - env : :class:`~.ComputeEnvironment` - The environment to use for submission. Uses the environment defined - by the :class:`~.FlowProject` if None (Default value = None). \*\*kwargs Additional keyword aruguments forwarded to :meth:`~.ComputeEnvironment.submit`. @@ -3725,6 +3725,9 @@ def submit_operations( The operations to submit. _id : str The _id to be used for this submission. (Default value = None) + env : :class:`~.ComputeEnvironment` + The environment to use for submission. Uses the environment defined + by the :class:`~.FlowProject` if None (Default value = None). parallel : bool Execute all bundled operations in parallel. (Default value = False) flags : list @@ -3740,9 +3743,6 @@ def submit_operations( show_template_help : bool Show information about available template variables and filters and exit. (Default value = False) - env : :class:`~.ComputeEnvironment` - The environment to use for submission. Uses the environment defined - by the :class:`~.FlowProject` if None (Default value = None). \*\*kwargs Additional keyword aruguments forwarded to :meth:`~.ComputeEnvironment.submit`. From 21a8933f6d0d5d555ce4cc750904fa765a9a4992 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 11 Dec 2020 11:34:00 -0600 Subject: [PATCH 32/34] Fix typo. --- flow/project.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flow/project.py b/flow/project.py index 3a37d4068..e70827f8b 100644 --- a/flow/project.py +++ b/flow/project.py @@ -3632,7 +3632,7 @@ def _submit_operations( Show information about available template variables and filters and exit. (Default value = False) \*\*kwargs - Additional keyword aruguments forwarded to :meth:`~.ComputeEnvironment.submit`. + Additional keyword arguments forwarded to :meth:`~.ComputeEnvironment.submit`. Returns ------- @@ -3744,7 +3744,7 @@ def submit_operations( Show information about available template variables and filters and exit. (Default value = False) \*\*kwargs - Additional keyword aruguments forwarded to :meth:`~.ComputeEnvironment.submit`. + Additional keyword arguments forwarded to :meth:`~.ComputeEnvironment.submit`. Returns ------- @@ -3810,7 +3810,7 @@ def submit( The environment to use for submission. Uses the environment defined by the :class:`~.FlowProject` if None (Default value = None). \*\*kwargs - Additional keyword aruguments forwarded to :meth:`~.ComputeEnvironment.submit`. + Additional keyword arguments forwarded to :meth:`~.ComputeEnvironment.submit`. """ aggregates = self._convert_aggregates_from_jobs(jobs) From 9abf5cdd172169f39b8c913ff39891c3c9390204 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 11 Dec 2020 15:53:12 -0600 Subject: [PATCH 33/34] Apply suggested changes. --- flow/project.py | 50 ++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/flow/project.py b/flow/project.py index e70827f8b..0976d30e7 100644 --- a/flow/project.py +++ b/flow/project.py @@ -262,7 +262,7 @@ def _make_bundles(operations, size=None): class _JobOperation: """Class containing execution information for one group and one job. - The execution or submission of a :class:`FlowGroup` uses a passed-in command + The execution or submission of a :class:`~.FlowGroup` uses a passed-in command which can either be a string or function with no arguments that returns a shell executable command. The shell executable command won't be used if it is determined that the group can be executed without forking. @@ -541,9 +541,9 @@ def __eq__(self, other): class BaseFlowOperation: - """A BaseFlowOperation represents a data space operation acting on any job. + """A :class:`~.BaseFlowOperation` represents a data space operation acting on any job. - Every BaseFlowOperation is associated with a specific command. + Every :class:`~.BaseFlowOperation` is associated with a specific command. Preconditions (pre) and postconditions (post) can be used to trigger an operation only when certain conditions are met. Conditions are unary @@ -839,10 +839,10 @@ def decorator(func): class FlowGroup: - """A FlowGroup represents a subset of a workflow for a project. + """A :class:`~.FlowGroup` represents a subset of a workflow for a project. A :class:`FlowGroup` is associated with one or more instances of - :class:`BaseFlowOperation`. + :class:`~.BaseFlowOperation`. Examples -------- @@ -871,7 +871,7 @@ def op2(job): The name of the group to be used when calling from the command line. operations : dict A dictionary of operations where the keys are operation names and - each value is a :class:`BaseFlowOperation`. + each value is a :class:`~.BaseFlowOperation`. operation_directives : dict A dictionary of additional parameters that provide instructions on how to execute a particular operation, e.g., specifically required @@ -968,8 +968,8 @@ def __repr__(self): def _eligible(self, jobs, ignore_conditions=IgnoreConditions.NONE): """Determine if at least one operation is eligible. - A :class:`~.FlowGroup` is eligible for execution if at least one of its - associated operations is eligible. + A :class:`~.FlowGroup` is eligible for execution if at least one of + its associated operations is eligible. Parameters ---------- @@ -1009,8 +1009,8 @@ def _complete(self, jobs): def eligible(self, job, ignore_conditions=IgnoreConditions.NONE): """Determine if at least one operation is eligible. - A FlowGroup is eligible for execution if at least one of its - associated operations is eligible. + A :class:`~.FlowGroup` is eligible for execution if at least one of + its associated operations is eligible. Parameters ---------- @@ -1031,7 +1031,7 @@ def eligible(self, job, ignore_conditions=IgnoreConditions.NONE): @deprecated(deprecated_in="0.11", removed_in="0.13", current_version=__version__) def complete(self, job): - """Check if all BaseFlowOperation postconditions are met. + """Check if all :class:`~.BaseFlowOperation` postconditions are met. Parameters ---------- @@ -1054,7 +1054,7 @@ def add_operation(self, name, operation, directives=None): ---------- name : str The name of the operation. - operation : :class:`BaseFlowOperation` + operation : :class:`~.BaseFlowOperation` The workflow operation to add to the :class:`~.FlowGroup`. directives : dict The operation specific directives. (Default value = None) @@ -1071,8 +1071,8 @@ def isdisjoint(self, group): Parameters ---------- - group : :class:`flow.project.FlowGroup` - The other FlowGroup to compare to. + group : :class:`~.FlowGroup` + The other :class:`~.FlowGroup` to compare to. Returns ------- @@ -1160,7 +1160,7 @@ def _create_submission_job_operation( ignore_conditions_on_execution=IgnoreConditions.NONE, index=0, ): - """Create a _JobOperation object from the FlowGroup. + """Create a _JobOperation object from the :class:`~.FlowGroup`. Creates a _JobOperation for use in submitting and scripting. @@ -1263,7 +1263,7 @@ def _create_run_job_operations( ignore_conditions=IgnoreConditions.NONE, index=0, ): - """Create _JobOperation object(s) from the FlowGroup. + """Create _JobOperation object(s) from the :class:`~.FlowGroup`. Yields a _JobOperation for each contained operation given proper conditions are met. @@ -3365,7 +3365,7 @@ def key_func_by_job(operation): ) def _gather_flow_groups(self, names=None): - """Grabs FlowGroups that match any of a set of names.""" + r"""Grabs :class:`~.FlowGroup`\ s that match any of a set of names.""" operations = {} # if no names are selected try all singleton groups if names is None: @@ -3400,7 +3400,7 @@ def _get_submission_operations( ignore_conditions=IgnoreConditions.NONE, ignore_conditions_on_execution=IgnoreConditions.NONE, ): - """Grabs _JobOperations that are eligible to run from FlowGroups.""" + r"""Grabs eligible :class:`~._JobOperation`\ s from :class:`~.FlowGroup`s.""" for group in self._gather_flow_groups(names): for aggregate in self._get_aggregate_store(group.name).values(): if ( @@ -3609,7 +3609,7 @@ def _submit_operations( Parameters ---------- - operations : A sequence of instances of :class:`._JobOperation` + operations : A sequence of instances of :class:`~._JobOperation` The operations to submit. _id : str The _id to be used for this submission. (Default value = None) @@ -4477,10 +4477,10 @@ def _register_aggregates(self): @classmethod def make_group(cls, name, options=""): - """Make a FlowGroup named ``name`` and return a decorator to make groups. + r"""Make a :class:`~.FlowGroup` named ``name`` and return a decorator to make groups. - FlowGroups group operations together for running and submitting - JobOperations. + A :class:`~.FlowGroup` is used to group operations together for + running and submitting :class:`~.JobOperation`\ s. Examples -------- @@ -4564,17 +4564,17 @@ def groups(self): return self._groups def _get_aggregate_store(self, group): - """Return aggregate store associated with the FlowGroup. + """Return aggregate store associated with the :class:`~.FlowGroup`. Parameters ---------- group : str - The name of the FlowGroup whose aggregate store will be returned. + The name of the :class:`~.FlowGroup` whose aggregate store will be returned. Returns ------- :class:`_DefaultAggregateStore` - Aggregate store containing aggregates associated with the provided FlowGroup. + Aggregate store containing aggregates associated with the provided :class:`~.FlowGroup`. """ for aggregate_store, groups in self._stored_aggregates.items(): From a371289217ca70c055c2f13ea807944317cbbdb0 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 11 Dec 2020 21:09:19 -0600 Subject: [PATCH 34/34] Add docs to simple-scheduler, don't git ignore the bin directory containing simple-scheduler. --- .gitignore | 1 - bin/simple-scheduler | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index fd0c55878..1c83d7059 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,6 @@ dist build eggs parts -bin var sdist develop-eggs diff --git a/bin/simple-scheduler b/bin/simple-scheduler index 613f90f3c..21e1985f9 100755 --- a/bin/simple-scheduler +++ b/bin/simple-scheduler @@ -1,4 +1,13 @@ #!/usr/bin/env python +# Copyright (c) 2020 The Regents of the University of Michigan +# All rights reserved. +# This software is licensed under the BSD 3-Clause License. +"""A simple scheduler implementation that works with signac-flow. + +The package signac-flow includes the ``simple-scheduler`` script as a simple +model of a cluster job scheduler. The ``simple-scheduler`` script is designed +primarily for testing and demonstration. +""" import argparse import json import logging @@ -41,6 +50,11 @@ def _get_args(script): def main_submit(args): + """Submit jobs. + + Jobs are accepted from a provided filename and copied to a unique name in + the scheduler's inbox. + """ # Try to parse args, should raise error if anything is wrong or missing with open(args.filename) as script: _get_submit_parser().parse_args(list(_get_args(script))) @@ -152,6 +166,11 @@ def _process_queue(args, db): def main_run(args): + """Run the scheduler. + + The scheduler runs multiple threads. A separate thread moves jobs from the + inbox to a queue. The main thread processes jobs in the queue. + """ print("Start scheduler...") os.makedirs(args.inbox, exist_ok=True) os.makedirs(args.queue, exist_ok=True) @@ -175,6 +194,10 @@ def main_run(args): def main_status(args): + """Get job status. + + This returns the status of all jobs from the scheduler's database. + """ with Collection.open(args.db, mode="r") as db: if args.json: print(json.dumps({doc["_id"]: doc for doc in db}, indent=4))