Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Linting dicts. #374

Merged
merged 3 commits into from
Dec 3, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions flow/aggregates.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@
# All rights reserved.
# This software is licensed under the BSD 3-Clause License.
import itertools
from collections import Mapping, OrderedDict
from collections.abc import Iterable
from collections.abc import Iterable, Mapping
from hashlib import md5


Expand Down Expand Up @@ -316,7 +315,7 @@ def __init__(self, aggregator, project):
# We need to register the aggregates for this instance using the
# project provided. After registering, we store the aggregates
# mapped with the ids using the `get_aggregate_id` method.
self._aggregate_per_id = OrderedDict()
self._aggregate_per_id = {}
self._register_aggregates(project)

def __iter__(self):
Expand Down
6 changes: 3 additions & 3 deletions flow/directives.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,9 +119,9 @@ class _Directives(MutableMapping):
"""

def __init__(self, environment_directives):
self._directive_definitions = dict()
self._defined_directives = dict()
self._user_directives = dict()
self._directive_definitions = {}
self._defined_directives = {}
self._user_directives = {}

for directive in environment_directives:
self._add_directive(directive)
Expand Down
5 changes: 2 additions & 3 deletions flow/environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
import os
import re
import socket
from collections import OrderedDict

from signac.common import config

Expand Down Expand Up @@ -67,7 +66,7 @@ def run(self):
msg=f"registering module '{name}' in global signac configuration",
level=2,
)
cfg.setdefault("flow", dict())
cfg.setdefault("flow", {})
cfg["flow"]["environment_modules"] = envs + list(new)
cfg.write()

Expand All @@ -85,7 +84,7 @@ class ComputeEnvironmentType(type):

def __init__(cls, name, bases, dct):
if not hasattr(cls, "registry"):
cls.registry = OrderedDict()
cls.registry = {}
else:
cls.registry[name] = cls
return super().__init__(name, bases, dct)
Expand Down
4 changes: 2 additions & 2 deletions flow/operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,10 +126,10 @@ def __init__(self, **kwargs):

@classmethod
def copy_from(cls, func):
return cls(**getattr(func, "_flow_directives", dict()))
return cls(**getattr(func, "_flow_directives", {}))

def __call__(self, func):
directives = getattr(func, "_flow_directives", dict())
directives = getattr(func, "_flow_directives", {})
directives.update(self.kwargs)
setattr(func, "_flow_directives", directives)
return func
Expand Down
156 changes: 81 additions & 75 deletions flow/project.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import time
import traceback
import warnings
from collections import Counter, OrderedDict, defaultdict
from collections import Counter, defaultdict
from copy import deepcopy
from enum import IntFlag
from hashlib import sha1
Expand Down Expand Up @@ -294,9 +294,7 @@ def __init__(self, id, name, jobs, cmd, directives=None):
# We use a special dictionary that allows us to track all keys that have been
# evaluated by the template engine and compare them to those explicitly set
# by the user. See also comment above.
self.directives = TrackGetItemDict(
{key: value for key, value in directives.items()}
)
self.directives = TrackGetItemDict(directives)
self.directives._keys_set_by_user = keys_set_by_user

def __str__(self):
Expand Down Expand Up @@ -851,13 +849,11 @@ def op2(job):
def __init__(self, name, operations=None, operation_directives=None, options=""):
self.name = name
self.options = options
# An OrderedDict is not necessary here, but is used to ensure
# consistent ordering of pretend submission output for templates.
self.operations = (
OrderedDict() if operations is None else OrderedDict(operations)
)
# Requires Python >=3.6: dict must be ordered to ensure consistent
# pretend submission output for templates.
self.operations = {} if operations is None else dict(operations)
if operation_directives is None:
self.operation_directives = dict()
self.operation_directives = {}
else:
self.operation_directives = operation_directives

Expand Down Expand Up @@ -1284,7 +1280,7 @@ def __new__(metacls, name, bases, namespace, **kwargs):
# All label functions are registered with the label() classmethod, which is intended
# to be used as decorator function. The _LABEL_FUNCTIONS dict contains the function as
# key and the label name as value, or None to use the default label name.
cls._LABEL_FUNCTIONS = OrderedDict()
cls._LABEL_FUNCTIONS = {}

# Give the class a pre and post class that are aware of the class they
# are in.
Expand Down Expand Up @@ -1477,11 +1473,11 @@ def __init__(self, config=None, environment=None, entrypoint=None):
self._template_environment_ = dict()

# Register all label functions with this project instance.
self._label_functions = OrderedDict()
self._label_functions = {}
self._register_labels()

# Register all operation functions with this project instance.
self._operations = OrderedDict()
self._operations = {}
self._register_operations()

# Register all groups with this project instance.
Expand Down Expand Up @@ -1868,8 +1864,8 @@ def get_job_status(self, job, ignore_errors=False, cached_status=None):
try:
cached_status = self.document["_status"]._as_dict()
except KeyError:
cached_status = dict()
result["operations"] = OrderedDict(
cached_status = {}
result["operations"] = dict(
self._get_operations_status((job,), cached_status)
)
result["_operations_error"] = None
Expand Down Expand Up @@ -2440,7 +2436,7 @@ def print_status(
]

with prof(single=False):
tmp = self._fetch_status(
fetched_status = self._fetch_status(
aggregates,
distinct_jobs,
err,
Expand All @@ -2454,9 +2450,9 @@ def print_status(
total_impact = 0
hits = [
hit
for fn, ft in prof.merged_file_dict.items()
for fn, file_timing in prof.merged_file_dict.items()
if fn not in fn_filter
for hit in ft.iterHits()
for hit in file_timing.iterHits()
]
sorted_hits = reversed(sorted(hits, key=lambda hit: hit[2]))
total_num_hits = sum([hit[2] for hit in hits])
Expand All @@ -2480,17 +2476,17 @@ def print_status(

for module_fn in prof.merged_file_dict:
if re.match(profile, module_fn):
ft = prof.merged_file_dict[module_fn]
file_timing = prof.merged_file_dict[module_fn]
else:
continue

total_hits = ft.getTotalHitCount()
total_hits = file_timing.getTotalHitCount()
total_impact = 0

profiling_results.append(f"\nHits by line for '{module_fn}':")
profiling_results.append("-" * len(profiling_results[-1]))

hits = list(sorted(ft.iterHits(), key=lambda h: 1 / h[2]))
hits = list(sorted(file_timing.iterHits(), key=lambda h: 1 / h[2]))
for line, code, hits, duration in hits:
impact = hits / total_hits
total_impact += impact
Expand All @@ -2500,7 +2496,7 @@ def print_status(
except OSError:
continue
hits_ = [
ft.getHitStatsFor(line)[0]
file_timing.getHitStatsFor(line)[0]
for line in range(start, start + len(lines))
]
profiling_results.extend(
Expand All @@ -2521,13 +2517,17 @@ def print_status(
)

else:
tmp = self._fetch_status(
fetched_status = self._fetch_status(
aggregates, distinct_jobs, err, ignore_errors, status_parallelization
)
profiling_results = None

operations_errors = {s["_operations_error"] for s in tmp}
labels_errors = {s["_labels_error"] for s in tmp}
operations_errors = {
status_entry["_operations_error"] for status_entry in fetched_status
}
labels_errors = {
status_entry["_labels_error"] for status_entry in fetched_status
}
errors = list(filter(None, operations_errors.union(labels_errors)))

if errors:
Expand All @@ -2541,21 +2541,25 @@ def print_status(
logger.debug("Status update error #{}: '{}'".format(i + 1, error))

if only_incomplete:
# Remove all jobs from the status info, that have not a single
# eligible operation.
# Remove jobs with no eligible operations from the status info

def _incomplete(s):
return any(op["eligible"] for op in s["operations"].values())
def _incomplete(status_entry):
return any(
operation["eligible"]
for operation in status_entry["operations"].values()
)

tmp = list(filter(_incomplete, tmp))
fetched_status = list(filter(_incomplete, fetched_status))

statuses = OrderedDict([(s["job_id"], s) for s in tmp])
statuses = {
status_entry["job_id"]: status_entry for status_entry in fetched_status
}

# If the dump_json variable is set, just dump all status info
# formatted in JSON to screen.
if dump_json:
print(json.dumps(statuses, indent=4), file=file)
return
return None

if overview:
# get overview info:
Expand Down Expand Up @@ -2590,21 +2594,23 @@ def _incomplete(s):
# get parameters info

def _add_parameters(status):
sp = self.open_job(id=status["job_id"]).statepoint()

def get(k, m):
if m is None:
return
t = k.split(".")
if len(t) > 1:
return get(".".join(t[1:]), m.get(t[0]))
else:
return m.get(k)

status["parameters"] = OrderedDict()
for i, k in enumerate(parameters):
v = shorten(str(self._alias(get(k, sp))), param_max_width)
status["parameters"][k] = v
statepoint = self.open_job(id=status["job_id"]).statepoint()

def dotted_get(mapping, key):
"""Fetch a value from a nested mapping using a dotted key."""
if mapping is None:
return None
tokens = key.split(".")
if len(tokens) > 1:
return dotted_get(mapping.get(tokens[0]), ".".join(tokens[1:]))
return mapping.get(key)

status["parameters"] = {}
for parameter in parameters:
status["parameters"][parameter] = shorten(
str(self._alias(dotted_get(statepoint, parameter))),
param_max_width,
)

for status in statuses.values():
_add_parameters(status)
Expand All @@ -2620,27 +2626,23 @@ def get(k, m):
num_operations = len(self._operations)

if pretty:
OPERATION_STATUS_SYMBOLS = OrderedDict(
[
("ineligible", "\u25cb"), # open circle
("eligible", "\u25cf"), # black circle
("active", "\u25b9"), # open triangle
("running", "\u25b8"), # black triangle
("completed", "\u2714"), # check mark
]
)
"Pretty (unicode) symbols denoting the execution status of operations."
OPERATION_STATUS_SYMBOLS = {
"ineligible": "\u25cb", # open circle
"eligible": "\u25cf", # black circle
"active": "\u25b9", # open triangle
"running": "\u25b8", # black triangle
"completed": "\u2714", # check mark
}
# Pretty (unicode) symbols denoting the execution status of operations.
else:
OPERATION_STATUS_SYMBOLS = OrderedDict(
[
("ineligible", "-"),
("eligible", "+"),
("active", "*"),
("running", ">"),
("completed", "X"),
]
)
"Symbols denoting the execution status of operations."
OPERATION_STATUS_SYMBOLS = {
"ineligible": "-",
"eligible": "+",
"active": "*",
"running": ">",
"completed": "X",
}
# ASCII symbols denoting the execution status of operations.
operation_status_legend = " ".join(
f"[{v}]:{k}" for k, v in OPERATION_STATUS_SYMBOLS.items()
)
Expand Down Expand Up @@ -3158,7 +3160,7 @@ def key_func_by_job(op):

def _gather_flow_groups(self, names=None):
"""Grabs FlowGroups that match any of a set of names."""
operations = OrderedDict()
operations = {}
# if no names are selected try all singleton groups
if names is None:
names = self._operations.keys()
Expand Down Expand Up @@ -4299,23 +4301,27 @@ def _register_groups(self):
self._aggregator_per_group = dict(aggregators)

# Add operations and directives to group
for (op_name, op) in self._operations.items():
if isinstance(op, FlowCmdOperation):
func = op._cmd
for operation_name, operation in self._operations.items():
if isinstance(operation, FlowCmdOperation):
func = operation._cmd
else:
func = op._op_func
func = operation._op_func

if hasattr(func, "_flow_groups"):
op_directives = getattr(
func, "_flow_group_operation_directives", dict()
)
for group_name in func._flow_groups:
directives = op_directives.get(group_name)
self._groups[group_name].add_operation(op_name, op, directives)
self._groups[group_name].add_operation(
operation_name, operation, directives
)

# For singleton groups add directives
directives = getattr(func, "_flow_directives", dict())
self._groups[op_name].operation_directives[op_name] = directives
directives = getattr(func, "_flow_directives", {})
self._groups[operation_name].operation_directives[
operation_name
] = directives

@property
def operations(self):
Expand Down
2 changes: 1 addition & 1 deletion flow/scheduling/status.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ def update_status(job, scheduler_jobs=None):
"""Update the job's status dictionary."""

# The status docs maps the scheduler job id to a distinct JobStatus value.
status_doc = job.document.setdefault("status", dict())
status_doc = job.document.setdefault("status", {})

# Iterate through all entries within the job's status doc:
for scheduler_job_id in status_doc:
Expand Down
4 changes: 2 additions & 2 deletions flow/template.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,12 @@ def init(alias=None, template=None, root=None, out=None):
trim_blocks=True,
)

context = dict()
context = {}
context["alias"] = alias
context["project_class_name"] = project_class_name

# render all templates
codes = dict()
codes = {}

for fn, fn_template in TEMPLATES[template]:
fn_ = fn.format(alias=alias) # some of the filenames may depend on the alias
Expand Down
Loading