From ec92a4a28a92e287f868842a1076e6d704d86cd8 Mon Sep 17 00:00:00 2001 From: David Young Date: Thu, 3 Jun 2021 15:41:46 +0800 Subject: [PATCH 1/3] Fix unicode error when attempting to load NPZ metadata files as YML files Loading metadata first attempts to load the file as YML format. If the metadata has not been updated to YML format, the load typically results in a file not found error because main image reader formats the metadata path as a `.yml` file. When the user specifies metadata directly through the `--meta` flag, however, the YML loader attempts to read the NPZ file and gives a Unicode decode error. Catch both file not found and unicode errors in the main YML loader function, raising a single file not found error, which the metadata loader already catches. Additionally, remove the unused Numpy import statement in the YML I/O module. --- magmap/io/yaml_io.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/magmap/io/yaml_io.py b/magmap/io/yaml_io.py index fb3f8d18e..d866a0b94 100644 --- a/magmap/io/yaml_io.py +++ b/magmap/io/yaml_io.py @@ -2,7 +2,6 @@ # Author: David Young, 2020 """YAML file format input/output.""" -import numpy as np import yaml from magmap.io import libmag @@ -49,6 +48,9 @@ def load_yaml(path, enums=None): Returns: List[dict]: Sequence of parsed dictionaries for each document within a YAML file. + + Raises: + FileNotFoundError: if ``path`` could not be found or loaded. """ def parse_enum_val(val): @@ -63,18 +65,21 @@ def parse_enum_val(val): # replace with the corresponding Enum class val = enums[val_split[0]][val_split[1]] return val - - with open(path) as yaml_file: - # load all documents into a generator - docs = yaml.load_all(yaml_file, Loader=yaml.FullLoader) - data = [] - for doc in docs: - if not doc: - # skip empty document - continue - if enums: - doc = _filter_dict(doc, parse_enum_val) - data.append(doc) + + try: + with open(path) as yaml_file: + # load all documents into a generator + docs = yaml.load_all(yaml_file, Loader=yaml.FullLoader) + data = [] + for doc in docs: + if not doc: + # skip empty document + continue + if enums: + doc = _filter_dict(doc, parse_enum_val) + data.append(doc) + except (FileNotFoundError, UnicodeDecodeError) as e: + raise FileNotFoundError(e) return data From 51ce6e3076b3e313170da1520e32ca15e494b38b Mon Sep 17 00:00:00 2001 From: David Young Date: Thu, 3 Jun 2021 15:50:40 +0800 Subject: [PATCH 2/3] Resave metadata loaded in NPZ format as YML files Image metadata loaded through the main reader as NPZ files have been automatically resaved in YML format, but metadata loaded outside of this reader are not resaved. Integrate the YML saving into the metadata loader to resave the metadata whenever it is loaded as NPZ format, even if the file will be resaved later after any metadata updates. Also, convert warning calls to logger calls for consistency. --- magmap/io/importer.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/magmap/io/importer.py b/magmap/io/importer.py index 494711931..9951392d2 100644 --- a/magmap/io/importer.py +++ b/magmap/io/importer.py @@ -541,16 +541,23 @@ def load_metadata(path, check_ver=False, assign=True): if output: # metadata is in first document output = output[0] - except FileNotFoundError: + except FileNotFoundError as err: # fall back to pre-v1.4 NPZ file format - libmag.warn("Could not find metadata file '{}', will check NPZ format" - .format(path)) + _logger.warn("Could not load metadata file '%s', will check NPZ format", + path) + _logger.debug(err) path_npz = f"{os.path.splitext(path)[0]}.npz" try: + # load NPZ and resave as YML format output = np_io.read_np_archive(np.load(path_npz)) - except FileNotFoundError: - libmag.warn("Could not find metadata file '{}', skipping" - .format(path_npz)) + path_yml = f"{os.path.splitext(path)[0]}.yml" + yaml_io.save_yaml(path_yml, output, True) + _logger.info( + "Metadata file from '%s' updated to '%s'", path_npz, path_yml) + except FileNotFoundError as err2: + _logger.warn( + "Could not load metadata file '%s', skipping", path_npz) + _logger.debug(err2) return None, image5d_ver_num try: From 959cf777a067e356f43c6b68d2fa9fcb4361ae37 Mon Sep 17 00:00:00 2001 From: David Young Date: Thu, 3 Jun 2021 15:54:11 +0800 Subject: [PATCH 3/3] Fix error when unable to load profile files Catch YML load errors when attempting to load profile files. Also, convert debugging print calls to logging calls. --- magmap/settings/profiles.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/magmap/settings/profiles.py b/magmap/settings/profiles.py index 2b6ba359b..8457d9cbf 100644 --- a/magmap/settings/profiles.py +++ b/magmap/settings/profiles.py @@ -14,6 +14,8 @@ from magmap.io import yaml_io from magmap.settings import config +_logger = config.logger.getChild(__name__) + class RegKeys(Enum): """Register setting enumerations.""" @@ -142,11 +144,14 @@ def add_modifier(self, mod_name, profiles, sep): print(mod_path, "profile file not found, skipped") return self.timestamps[mod_path] = os.path.getmtime(mod_path) - yamls = yaml_io.load_yaml(mod_path, _PROFILE_ENUMS) - mods = {} - for yaml in yamls: - mods.update(yaml) - print("loaded {}:\n{}".format(mod_path, mods)) + try: + yamls = yaml_io.load_yaml(mod_path, _PROFILE_ENUMS) + mods = {} + for yaml in yamls: + mods.update(yaml) + _logger.info("Loaded profile from '%s':\n%s", mod_path, mods) + except FileNotFoundError: + _logger.warn("Unable to load profile from: %s", mod_path) else: if mod_name == self.DEFAULT_NAME: # update entries from a new instance for default values; @@ -194,8 +199,8 @@ def update_settings(self, names_str): self.add_modifier(profile, self.profiles, self.delimiter) if config.verbose: - print("settings for {}:", self[self.NAME_KEY]) - pprint.pprint(self) + _logger.debug("settings for '%s':", self[self.NAME_KEY]) + _logger.debug(pprint.pprint(self)) def check_file_changed(self): """Check whether any profile files have changed since last loaded.