From a3bda95a09d1d2a923dcb3a99f1541ba36a55fe9 Mon Sep 17 00:00:00 2001 From: Jason Munro Date: Thu, 1 Sep 2022 14:45:18 -0700 Subject: [PATCH 01/14] Make custodian and mpcontribs optional deps --- mp_api/client/client.py | 2 +- mp_api/client/core/client.py | 3 +- mp_api/client/core/utils.py | 93 +++++++++++++++++++++++++++++++++++- setup.py | 11 +++-- 4 files changed, 101 insertions(+), 8 deletions(-) diff --git a/mp_api/client/client.py b/mp_api/client/client.py index 7b7bc6f9..35397b3d 100644 --- a/mp_api/client/client.py +++ b/mp_api/client/client.py @@ -11,7 +11,6 @@ from emmet.core.summary import HasProps from emmet.core.symmetry import CrystalSystem from emmet.core.vasp.calc_types import CalcType -from mpcontribs.client import Client from pymatgen.analysis.magnetism import Ordering from pymatgen.analysis.phase_diagram import PhaseDiagram from pymatgen.analysis.pourbaix_diagram import IonEntry @@ -133,6 +132,7 @@ def __init__( ) try: + from mpcontribs.client import Client self.contribs = Client(api_key) except Exception as error: self.contribs = None diff --git a/mp_api/client/core/client.py b/mp_api/client/core/client.py index 83516c42..137b7969 100644 --- a/mp_api/client/core/client.py +++ b/mp_api/client/core/client.py @@ -19,7 +19,6 @@ import requests from emmet.core.utils import jsanitize -from maggma.api.utils import api_sanitize from monty.json import MontyDecoder from pydantic import BaseModel, create_model from requests.adapters import HTTPAdapter @@ -28,7 +27,7 @@ from urllib3.util.retry import Retry from mp_api.client.core.settings import MAPIClientSettings -from mp_api.client.core.utils import validate_ids +from mp_api.client.core.utils import validate_ids, api_sanitize try: from pymatgen.core import __version__ as pmg_version # type: ignore diff --git a/mp_api/client/core/utils.py b/mp_api/client/core/utils.py index 6531c98d..8966a189 100644 --- a/mp_api/client/core/utils.py +++ b/mp_api/client/core/utils.py @@ -1,5 +1,10 @@ import re -from typing import List +from typing import List, Optional, Type, get_args + +from monty.json import MSONable +from pydantic import BaseModel +from pydantic.schema import get_flat_models_from_model +from pydantic.utils import lenient_issubclass def validate_ids(id_list: List[str]): @@ -21,3 +26,89 @@ def validate_ids(id_list: List[str]): raise ValueError(f"{entry} is not formatted correctly!") return id_list + + +def api_sanitize( + pydantic_model: Type[BaseModel], + fields_to_leave: Optional[List[str]] = None, + allow_dict_msonable=False, +): + """ + Function to clean up pydantic models for the API by: + 1.) Making fields optional + 2.) Allowing dictionaries in-place of the objects for MSONable quantities + + WARNING: This works in place, so it mutates the model and all sub-models + + Args: + fields_to_leave: list of strings for model fields as "model__name__.field" + """ + + models = [ + model + for model in get_flat_models_from_model(pydantic_model) + if issubclass(model, BaseModel) + ] # type: List[Type[BaseModel]] + + fields_to_leave = fields_to_leave or [] + fields_tuples = [f.split(".") for f in fields_to_leave] + assert all(len(f) == 2 for f in fields_tuples) + + for model in models: + model_fields_to_leave = {f[1] for f in fields_tuples if model.__name__ == f[0]} + for name, field in model.__fields__.items(): + field_type = field.type_ + + if name not in model_fields_to_leave: + field.required = False + field.default = None + field.default_factory = None + field.allow_none = True + field.field_info.default = None + field.field_info.default_factory = None + + if field_type is not None and allow_dict_msonable: + if lenient_issubclass(field_type, MSONable): + field.type_ = allow_msonable_dict(field_type) + else: + for sub_type in get_args(field_type): + if lenient_issubclass(sub_type, MSONable): + allow_msonable_dict(sub_type) + field.populate_validators() + + return pydantic_model + + +def allow_msonable_dict(monty_cls: Type[MSONable]): + """ + Patch Monty to allow for dict values for MSONable + """ + + def validate_monty(cls, v): + """ + Stub validator for MSONable as a dictionary only + """ + if isinstance(v, cls): + return v + elif isinstance(v, dict): + # Just validate the simple Monty Dict Model + errors = [] + if v.get("@module", "") != monty_cls.__module__: + errors.append("@module") + + if v.get("@class", "") != monty_cls.__name__: + errors.append("@class") + + if len(errors) > 0: + raise ValueError( + "Missing Monty seriailzation fields in dictionary: {errors}" + ) + + return v + else: + raise ValueError(f"Must provide {cls.__name__} or MSONable dictionary") + + setattr(monty_cls, "validate_monty", classmethod(validate_monty)) + + return monty_cls + diff --git a/setup.py b/setup.py index 9ce93c26..4fd41741 100644 --- a/setup.py +++ b/setup.py @@ -28,11 +28,14 @@ "typing-extensions>=3.7.4.1", "requests>=2.23.0", "monty>=2021.3.12", - "emmet-core>=0.32.3", - "maggma>=0.47.4", - "custodian", - "mpcontribs-client", + "emmet-core>=0.35.0", ], + extras_require={ + "all": [ + "custodian", + "mpcontribs-client", + ], + }, classifiers=[ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", From dd55a189e96965d583b582fcf494ff0e3e65dd82 Mon Sep 17 00:00:00 2001 From: Jason Munro Date: Thu, 1 Sep 2022 14:48:02 -0700 Subject: [PATCH 02/14] Add emmet-core[all] to options --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 4fd41741..e918a7be 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ ], extras_require={ "all": [ + "emmet-core[all]>=0.35.0" "custodian", "mpcontribs-client", ], From 84621cef6e4a5f9b545ca697dec445b38d910f04 Mon Sep 17 00:00:00 2001 From: Jason Munro Date: Thu, 1 Sep 2022 15:45:33 -0700 Subject: [PATCH 03/14] Handle alloy rester import if no addon pkg --- mp_api/client/routes/__init__.py | 11 +++++++++-- mp_api/client/routes/alloys.py | 2 -- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/mp_api/client/routes/__init__.py b/mp_api/client/routes/__init__.py index c03796de..eb668185 100644 --- a/mp_api/client/routes/__init__.py +++ b/mp_api/client/routes/__init__.py @@ -1,3 +1,4 @@ +from ast import Import from .eos import EOSRester from .materials import MaterialsRester from .similarity import SimilarityRester @@ -15,7 +16,6 @@ from .piezo import PiezoRester from .magnetism import MagnetismRester from .summary import SummaryRester -from .robocrys import RobocrysRester from .molecules import MoleculesRester from .synthesis import SynthesisRester from .electrodes import ElectrodeRester @@ -30,4 +30,11 @@ from ._user_settings import UserSettingsRester from ._general_store import GeneralStoreRester from .bonds import BondsRester -from .alloys import AlloysRester +from .robocrys import RobocrysRester + +try: + from .alloys import AlloysRester +except ImportError: + import warnings + warnings.warn("Alloy addon package not installed. To query alloy data install the pymatgen-analysis-alloys package.") + AlloysRester = None # type: ignore diff --git a/mp_api/client/routes/alloys.py b/mp_api/client/routes/alloys.py index 01612ebd..431e0af8 100644 --- a/mp_api/client/routes/alloys.py +++ b/mp_api/client/routes/alloys.py @@ -5,8 +5,6 @@ from mp_api.client.core.utils import validate_ids from emmet.core.alloys import AlloyPairDoc -import warnings - class AlloysRester(BaseRester[AlloyPairDoc]): From 6c4deed659e7388ef2c9cb6d8a7c692fccfcf7ff Mon Sep 17 00:00:00 2001 From: Jason Munro Date: Thu, 1 Sep 2022 15:49:17 -0700 Subject: [PATCH 04/14] Update requirements.txt --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 62c4e620..d74fa893 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,8 @@ -emmet-core==0.32.3 +emmet-core[all]==0.35.0 pydantic>=1.8.2 pymatgen>=2022.3.7 pymatgen-analysis-alloys>=0.0.3 typing-extensions==4.1.1 -maggma==0.47.4 requests==2.27.1 monty==2022.3.12 mpcontribs-client>=4.2.9 From e4863961fa1a1e55087a07341effae97eb0f17b8 Mon Sep 17 00:00:00 2001 From: Jason Munro Date: Thu, 1 Sep 2022 15:55:23 -0700 Subject: [PATCH 05/14] Linting --- mp_api/client/core/utils.py | 1 - mp_api/client/routes/__init__.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mp_api/client/core/utils.py b/mp_api/client/core/utils.py index 8966a189..0433217f 100644 --- a/mp_api/client/core/utils.py +++ b/mp_api/client/core/utils.py @@ -111,4 +111,3 @@ def validate_monty(cls, v): setattr(monty_cls, "validate_monty", classmethod(validate_monty)) return monty_cls - diff --git a/mp_api/client/routes/__init__.py b/mp_api/client/routes/__init__.py index eb668185..8f937509 100644 --- a/mp_api/client/routes/__init__.py +++ b/mp_api/client/routes/__init__.py @@ -36,5 +36,6 @@ from .alloys import AlloysRester except ImportError: import warnings - warnings.warn("Alloy addon package not installed. To query alloy data install the pymatgen-analysis-alloys package.") + warnings.warn("Alloy addon package not installed. " + "To query alloy data install the pymatgen-analysis-alloys package.") AlloysRester = None # type: ignore From 73a08e0d41d40f4d8366722539611642f85a0021 Mon Sep 17 00:00:00 2001 From: Jason Munro Date: Thu, 1 Sep 2022 16:53:27 -0700 Subject: [PATCH 06/14] Bump emmet-core --- requirements.txt | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index d74fa893..9e108c80 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -emmet-core[all]==0.35.0 +emmet-core[all]==0.35.1 pydantic>=1.8.2 pymatgen>=2022.3.7 pymatgen-analysis-alloys>=0.0.3 diff --git a/setup.py b/setup.py index e918a7be..b33038bb 100644 --- a/setup.py +++ b/setup.py @@ -28,11 +28,11 @@ "typing-extensions>=3.7.4.1", "requests>=2.23.0", "monty>=2021.3.12", - "emmet-core>=0.35.0", + "emmet-core>=0.35.1", ], extras_require={ "all": [ - "emmet-core[all]>=0.35.0" + "emmet-core[all]>=0.35.1" "custodian", "mpcontribs-client", ], From 03852bdda8a2925a890da5590156f9105da98856 Mon Sep 17 00:00:00 2001 From: Jason Munro Date: Thu, 1 Sep 2022 17:54:28 -0700 Subject: [PATCH 07/14] Add boto3 to optional deps and handle chg densities --- mp_api/client/routes/__init__.py | 9 ++++++++- setup.py | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/mp_api/client/routes/__init__.py b/mp_api/client/routes/__init__.py index 8f937509..6d3a3e9f 100644 --- a/mp_api/client/routes/__init__.py +++ b/mp_api/client/routes/__init__.py @@ -19,7 +19,6 @@ from .molecules import MoleculesRester from .synthesis import SynthesisRester from .electrodes import ElectrodeRester -from .charge_density import ChargeDensityRester from .electronic_structure import ( ElectronicStructureRester, BandStructureRester, @@ -39,3 +38,11 @@ warnings.warn("Alloy addon package not installed. " "To query alloy data install the pymatgen-analysis-alloys package.") AlloysRester = None # type: ignore + +try: + from .charge_density import ChargeDensityRester +except ImportError: + import warnings + warnings.warn("boto3 not installed. " + "To query charge density data install the boto3 package.") + ChargeDensityRester = None # type: ignore diff --git a/setup.py b/setup.py index b33038bb..12ddf282 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,7 @@ "emmet-core[all]>=0.35.1" "custodian", "mpcontribs-client", + "boto3" ], }, classifiers=[ From 0c5b96f1a7be273f24a0cb411a746c341ad6ab1f Mon Sep 17 00:00:00 2001 From: Jason Munro Date: Thu, 1 Sep 2022 18:01:25 -0700 Subject: [PATCH 08/14] Add boto3 to reqs list --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 9e108c80..d2fe4106 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,5 @@ requests==2.27.1 monty==2022.3.12 mpcontribs-client>=4.2.9 custodian +boto3 From 08b30739cff6de8bd00bdbb128b38bef85fb5c12 Mon Sep 17 00:00:00 2001 From: Jason Munro Date: Thu, 1 Sep 2022 21:10:42 -0700 Subject: [PATCH 09/14] Check for charge density rester in convenience fn --- mp_api/client/client.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mp_api/client/client.py b/mp_api/client/client.py index 35397b3d..85840a77 100644 --- a/mp_api/client/client.py +++ b/mp_api/client/client.py @@ -939,6 +939,10 @@ def get_charge_density_from_material_id( Returns: chgcar: Pymatgen Chgcar object. """ + + if not hasattr(self, "charge_density"): + raise MPRestError("boto3 not installed. " + "To query charge density data install the boto3 package.") # TODO: really we want a recommended task_id for charge densities here # this could potentially introduce an ambiguity From 5fa7a7750749e7379fc87c4989cbea04e2947892 Mon Sep 17 00:00:00 2001 From: Jason Munro Date: Thu, 1 Sep 2022 21:23:23 -0700 Subject: [PATCH 10/14] Custom __getattr__ for alloy and charge_density --- mp_api/client/client.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mp_api/client/client.py b/mp_api/client/client.py index 85840a77..da112b60 100644 --- a/mp_api/client/client.py +++ b/mp_api/client/client.py @@ -177,6 +177,18 @@ def __exit__(self, exc_type, exc_val, exc_tb): """ self.session.close() + def __getattr__(self, attr): + if attr == "alloys": + raise MPRestError("Alloy addon package not installed. " + "To query alloy data install the pymatgen-analysis-alloys package.") + elif attr == "charge_density": + raise MPRestError("boto3 not installed. " + "To query charge density data install the boto3 package.") + else: + raise AttributeError( + f"{self.__class__.__name__!r} object has no attribute {attr!r}" + ) + def get_task_ids_associated_with_material_id( self, material_id: str, calc_types: Optional[List[CalcType]] = None ) -> List[str]: From 10acf36d614c7e6e56c83bd390e5350cc5316856 Mon Sep 17 00:00:00 2001 From: Jason Munro Date: Thu, 1 Sep 2022 21:28:53 -0700 Subject: [PATCH 11/14] Linting --- mp_api/client/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mp_api/client/client.py b/mp_api/client/client.py index da112b60..6f0ee1c6 100644 --- a/mp_api/client/client.py +++ b/mp_api/client/client.py @@ -951,7 +951,7 @@ def get_charge_density_from_material_id( Returns: chgcar: Pymatgen Chgcar object. """ - + if not hasattr(self, "charge_density"): raise MPRestError("boto3 not installed. " "To query charge density data install the boto3 package.") From f7f6f2d3f00fa4c2ace425838f33779d938af630 Mon Sep 17 00:00:00 2001 From: Jason Munro Date: Tue, 6 Sep 2022 12:41:29 -0700 Subject: [PATCH 12/14] Separate import error and add pip commands --- mp_api/client/client.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mp_api/client/client.py b/mp_api/client/client.py index 6f0ee1c6..5dbece49 100644 --- a/mp_api/client/client.py +++ b/mp_api/client/client.py @@ -134,6 +134,10 @@ def __init__( try: from mpcontribs.client import Client self.contribs = Client(api_key) + except ImportError: + self.contribs = None + warnings.warn("mpcontribs-client not installed. " + "Install the package o query MPContribs data, or construct pourbaix diagrams.") except Exception as error: self.contribs = None warnings.warn(f"Problem loading MPContribs client: {error}") @@ -180,10 +184,10 @@ def __exit__(self, exc_type, exc_val, exc_tb): def __getattr__(self, attr): if attr == "alloys": raise MPRestError("Alloy addon package not installed. " - "To query alloy data install the pymatgen-analysis-alloys package.") + "To query alloy data first install with: 'pip install pymatgen-analysis-alloys'") elif attr == "charge_density": raise MPRestError("boto3 not installed. " - "To query charge density data install the boto3 package.") + "To query charge density data first install with: 'pip install boto3'") else: raise AttributeError( f"{self.__class__.__name__!r} object has no attribute {attr!r}" From aedd2f8196d6b4eee3e64d9132f1ffeec0af82ae Mon Sep 17 00:00:00 2001 From: Jason Munro Date: Tue, 6 Sep 2022 12:41:37 -0700 Subject: [PATCH 13/14] Remove instantiation warnings for client --- mp_api/client/routes/__init__.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/mp_api/client/routes/__init__.py b/mp_api/client/routes/__init__.py index 6d3a3e9f..d1fc9f69 100644 --- a/mp_api/client/routes/__init__.py +++ b/mp_api/client/routes/__init__.py @@ -34,15 +34,9 @@ try: from .alloys import AlloysRester except ImportError: - import warnings - warnings.warn("Alloy addon package not installed. " - "To query alloy data install the pymatgen-analysis-alloys package.") AlloysRester = None # type: ignore try: from .charge_density import ChargeDensityRester except ImportError: - import warnings - warnings.warn("boto3 not installed. " - "To query charge density data install the boto3 package.") ChargeDensityRester = None # type: ignore From f00c427327341c8e7be1755db4f3edeaa40eeb4a Mon Sep 17 00:00:00 2001 From: Jason Munro Date: Tue, 6 Sep 2022 12:58:04 -0700 Subject: [PATCH 14/14] Fix mpcontribs error message --- mp_api/client/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mp_api/client/client.py b/mp_api/client/client.py index 5dbece49..6b348285 100644 --- a/mp_api/client/client.py +++ b/mp_api/client/client.py @@ -137,7 +137,8 @@ def __init__( except ImportError: self.contribs = None warnings.warn("mpcontribs-client not installed. " - "Install the package o query MPContribs data, or construct pourbaix diagrams.") + "Install the package to query MPContribs data, or construct pourbaix diagrams: " + "'pip install mpcontribs-client'") except Exception as error: self.contribs = None warnings.warn(f"Problem loading MPContribs client: {error}")