diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6320280dd..8068f4ba1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,6 +60,7 @@ jobs: pip install -U setuptools wheel pip install -e . pip install -r requirements.txt + pip install -r requirements-server.txt pip install -r requirements-dev.txt - name: Run pre-commit @@ -85,6 +86,7 @@ jobs: pip install -U setuptools wheel pip install -e . pip install -r requirements.txt + pip install -r requirements-server.txt pip install -r requirements-dev.txt - name: Pass generated OpenAPI schemas through validator.swagger.io @@ -194,25 +196,29 @@ jobs: pip install -r requirements-dev.txt pip install -r requirements-http-client.txt - - name: Run all tests (using `mongomock`) - run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml tests/ + - name: Run non-server tests + run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml tests/ --ignore tests/server + + - name: Install latest server dependencies + run: pip install -r requirements-server.txt + + - name: Run server tests (using `mongomock`) + run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server tests/filtertransformers env: OPTIMADE_DATABASE_BACKEND: 'mongomock' - - name: Run server tests (using a real MongoDB) - run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server + run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server tests/filtertransformers env: OPTIMADE_DATABASE_BACKEND: 'mongodb' - name: Run server tests (using Elasticsearch) - run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server + run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server tests/filtertransformers env: OPTIMADE_DATABASE_BACKEND: 'elastic' - name: Install adapter conversion dependencies - run: | - pip install -r requirements-client.txt + run: pip install -r requirements-client.txt - name: Setup environment for AiiDA env: @@ -305,6 +311,7 @@ jobs: pip install -U setuptools wheel pip install -e . pip install -r requirements.txt + pip install -r requirements-server.txt pip install -r requirements-dev.txt pip install -r requirements-http-client.txt pip install -r requirements-docs.txt diff --git a/docs/api_reference/exceptions.md b/docs/api_reference/exceptions.md new file mode 100644 index 000000000..da8d3b8ed --- /dev/null +++ b/docs/api_reference/exceptions.md @@ -0,0 +1,3 @@ +# exceptions + +::: optimade.exceptions diff --git a/docs/api_reference/utils.md b/docs/api_reference/utils.md new file mode 100644 index 000000000..e227d8708 --- /dev/null +++ b/docs/api_reference/utils.md @@ -0,0 +1,3 @@ +# utils + +::: optimade.utils diff --git a/docs/api_reference/warnings.md b/docs/api_reference/warnings.md new file mode 100644 index 000000000..7da06b487 --- /dev/null +++ b/docs/api_reference/warnings.md @@ -0,0 +1,3 @@ +# warnings + +::: optimade.warnings diff --git a/optimade/adapters/warnings.py b/optimade/adapters/warnings.py index ad21a6e72..c40c11bbb 100644 --- a/optimade/adapters/warnings.py +++ b/optimade/adapters/warnings.py @@ -1,4 +1,4 @@ -from optimade.server.warnings import OptimadeWarning +from optimade.warnings import OptimadeWarning __all__ = ("AdapterPackageNotFound", "ConversionWarning") diff --git a/optimade/client/client.py b/optimade/client/client.py index c173c75db..64f53c333 100644 --- a/optimade/client/client.py +++ b/optimade/client/client.py @@ -34,8 +34,8 @@ TooManyRequestsException, silent_raise, ) +from optimade.exceptions import BadRequest from optimade.filterparser import LarkParser -from optimade.server.exceptions import BadRequest from optimade.utils import get_all_databases ENDPOINTS = ("structures", "references", "calculations", "info", "extensions") diff --git a/optimade/exceptions.py b/optimade/exceptions.py new file mode 100644 index 000000000..2a75cfd5b --- /dev/null +++ b/optimade/exceptions.py @@ -0,0 +1,115 @@ +from abc import ABC +from typing import Any, Dict, Optional, Tuple, Type + +__all__ = ( + "OptimadeHTTPException", + "BadRequest", + "VersionNotSupported", + "Forbidden", + "NotFound", + "UnprocessableEntity", + "InternalServerError", + "NotImplementedResponse", + "POSSIBLE_ERRORS", +) + + +class OptimadeHTTPException(Exception, ABC): + """This abstract class can be subclassed to define + HTTP responses with the desired status codes, and + detailed error strings to represent in the JSON:API + error response. + + This class closely follows the `starlette.HTTPException` without + requiring it as a dependency, so that such errors can also be + raised from within client code. + + Attributes: + status_code: The HTTP status code accompanying this exception. + title: A descriptive title for this exception. + detail: An optional string containing the details of the error. + + """ + + status_code: int + title: str + detail: Optional[str] = None + headers: Optional[Dict[str, Any]] = None + + def __init__( + self, detail: Optional[str] = None, headers: Optional[dict] = None + ) -> None: + if self.status_code is None: + raise AttributeError( + "HTTPException class {self.__class__.__name__} is missing required `status_code` attribute." + ) + self.detail = detail + self.headers = headers + + def __str__(self) -> str: + return self.detail if self.detail is not None else self.__repr__() + + def __repr__(self) -> str: + class_name = self.__class__.__name__ + return f"{class_name}(status_code={self.status_code!r}, detail={self.detail!r})" + + +class BadRequest(OptimadeHTTPException): + """400 Bad Request""" + + status_code: int = 400 + title: str = "Bad Request" + + +class VersionNotSupported(OptimadeHTTPException): + """553 Version Not Supported""" + + status_code: int = 553 + title: str = "Version Not Supported" + + +class Forbidden(OptimadeHTTPException): + """403 Forbidden""" + + status_code: int = 403 + title: str = "Forbidden" + + +class NotFound(OptimadeHTTPException): + """404 Not Found""" + + status_code: int = 404 + title: str = "Not Found" + + +class UnprocessableEntity(OptimadeHTTPException): + """422 Unprocessable Entity""" + + status_code: int = 422 + title: str = "Unprocessable Entity" + + +class InternalServerError(OptimadeHTTPException): + """500 Internal Server Error""" + + status_code: int = 500 + title: str = "Internal Server Error" + + +class NotImplementedResponse(OptimadeHTTPException): + """501 Not Implemented""" + + status_code: int = 501 + title: str = "Not Implemented" + + +"""A tuple of the possible errors that can be returned by an OPTIMADE API.""" +POSSIBLE_ERRORS: Tuple[Type[OptimadeHTTPException], ...] = ( + BadRequest, + Forbidden, + NotFound, + UnprocessableEntity, + InternalServerError, + NotImplementedResponse, + VersionNotSupported, +) diff --git a/optimade/filterparser/lark_parser.py b/optimade/filterparser/lark_parser.py index 775f1a315..01d5044cd 100644 --- a/optimade/filterparser/lark_parser.py +++ b/optimade/filterparser/lark_parser.py @@ -9,7 +9,7 @@ from lark import Lark, Tree -from optimade.server.exceptions import BadRequest +from optimade.exceptions import BadRequest __all__ = ("ParserError", "LarkParser") diff --git a/optimade/filtertransformers/base_transformer.py b/optimade/filtertransformers/base_transformer.py index 362dabce6..2bb89086c 100644 --- a/optimade/filtertransformers/base_transformer.py +++ b/optimade/filtertransformers/base_transformer.py @@ -11,9 +11,9 @@ from lark import Transformer, Tree, v_args -from optimade.server.exceptions import BadRequest +from optimade.exceptions import BadRequest from optimade.server.mappers import BaseResourceMapper -from optimade.server.warnings import UnknownProviderProperty +from optimade.warnings import UnknownProviderProperty __all__ = ( "BaseTransformer", diff --git a/optimade/filtertransformers/mongo.py b/optimade/filtertransformers/mongo.py index bcf18da25..862e21f5e 100755 --- a/optimade/filtertransformers/mongo.py +++ b/optimade/filtertransformers/mongo.py @@ -11,9 +11,9 @@ from lark import Token, v_args +from optimade.exceptions import BadRequest from optimade.filtertransformers.base_transformer import BaseTransformer, Quantity -from optimade.server.exceptions import BadRequest -from optimade.server.warnings import TimestampNotRFCCompliant +from optimade.warnings import TimestampNotRFCCompliant __all__ = ("MongoTransformer",) diff --git a/optimade/models/structures.py b/optimade/models/structures.py index 14262ecef..c9424254b 100644 --- a/optimade/models/structures.py +++ b/optimade/models/structures.py @@ -19,7 +19,7 @@ StrictField, SupportLevel, ) -from optimade.server.warnings import MissingExpectedField +from optimade.warnings import MissingExpectedField EXTENDED_CHEMICAL_SYMBOLS = set(CHEMICAL_SYMBOLS + EXTRA_SYMBOLS) diff --git a/optimade/server/entry_collections/entry_collections.py b/optimade/server/entry_collections/entry_collections.py index de5860a31..9b307672f 100644 --- a/optimade/server/entry_collections/entry_collections.py +++ b/optimade/server/entry_collections/entry_collections.py @@ -5,13 +5,13 @@ from lark import Transformer +from optimade.exceptions import BadRequest, Forbidden, NotFound from optimade.filterparser import LarkParser from optimade.models.entries import EntryResource from optimade.server.config import CONFIG, SupportedBackend -from optimade.server.exceptions import BadRequest, Forbidden, NotFound from optimade.server.mappers import BaseResourceMapper from optimade.server.query_params import EntryListingQueryParams, SingleEntryQueryParams -from optimade.server.warnings import ( +from optimade.warnings import ( FieldValueNotRecognized, QueryParamNotUsed, UnknownProviderProperty, diff --git a/optimade/server/exception_handlers.py b/optimade/server/exception_handlers.py index bb79e1436..06fc083a8 100644 --- a/optimade/server/exception_handlers.py +++ b/optimade/server/exception_handlers.py @@ -1,5 +1,5 @@ import traceback -from typing import Callable, Iterable, List, Optional, Tuple, Type +from typing import Callable, Iterable, List, Optional, Tuple, Type, Union from fastapi import Request from fastapi.encoders import jsonable_encoder @@ -7,9 +7,9 @@ from lark.exceptions import VisitError from pydantic import ValidationError +from optimade.exceptions import BadRequest, OptimadeHTTPException from optimade.models import ErrorResponse, ErrorSource, OptimadeError from optimade.server.config import CONFIG -from optimade.server.exceptions import BadRequest from optimade.server.logger import LOGGER from optimade.server.routers.utils import JSONAPIResponse, meta_values @@ -78,7 +78,8 @@ def general_exception( def http_exception_handler( - request: Request, exc: StarletteHTTPException + request: Request, + exc: Union[StarletteHTTPException, OptimadeHTTPException], ) -> JSONAPIResponse: """Handle a general HTTP Exception from Starlette @@ -152,7 +153,7 @@ def grammar_not_implemented_handler( All errors raised during filter transformation are wrapped in the Lark `VisitError`. According to the OPTIMADE specification, these errors are repurposed to be 501 NotImplementedErrors. - For special exceptions, like [`BadRequest`][optimade.server.exceptions.BadRequest], we pass-through to + For special exceptions, like [`BadRequest`][optimade.exceptions.BadRequest], we pass-through to [`general_exception()`][optimade.server.exception_handlers.general_exception], since they should not return a 501 NotImplementedError. @@ -226,6 +227,7 @@ def general_exception_handler(request: Request, exc: Exception) -> JSONAPIRespon ] ] = [ (StarletteHTTPException, http_exception_handler), + (OptimadeHTTPException, http_exception_handler), (RequestValidationError, request_validation_exception_handler), (ValidationError, validation_exception_handler), (VisitError, grammar_not_implemented_handler), diff --git a/optimade/server/exceptions.py b/optimade/server/exceptions.py index 88e41709b..995cfd8c8 100644 --- a/optimade/server/exceptions.py +++ b/optimade/server/exceptions.py @@ -1,105 +1,25 @@ -from abc import ABC -from typing import Optional +"""Reproduced imports from `optimade.exceptions` for backwards-compatibility.""" -from fastapi import HTTPException as FastAPIHTTPException +from optimade.exceptions import ( + POSSIBLE_ERRORS, + BadRequest, + Forbidden, + InternalServerError, + NotFound, + NotImplementedResponse, + OptimadeHTTPException, + UnprocessableEntity, + VersionNotSupported, +) __all__ = ( + "OptimadeHTTPException", "BadRequest", "VersionNotSupported", "Forbidden", "NotFound", "UnprocessableEntity", + "InternalServerError", "NotImplementedResponse", "POSSIBLE_ERRORS", ) - - -class HTTPException(FastAPIHTTPException, ABC): - """This abstract class makes it easier to subclass FastAPI's HTTPException with - new status codes. - - It can also be useful when testing requires a string representation - of an exception that contains the HTTPException - detail string, rather than the standard Python exception message. - - Attributes: - status_code: The HTTP status code accompanying this exception. - title: A descriptive title for this exception. - - """ - - status_code: int - title: str - - def __init__( - self, detail: Optional[str] = None, headers: Optional[dict] = None - ) -> None: - if self.status_code is None: - raise AttributeError( - "HTTPException class {self.__class__.__name__} is missing required `status_code` attribute." - ) - - super().__init__(status_code=self.status_code, detail=detail, headers=headers) - - def __str__(self) -> str: - return self.detail if self.detail is not None else self.__repr__() - - -class BadRequest(HTTPException): - """400 Bad Request""" - - status_code: int = 400 - title: str = "Bad Request" - - -class VersionNotSupported(HTTPException): - """553 Version Not Supported""" - - status_code: int = 553 - title: str = "Version Not Supported" - - -class Forbidden(HTTPException): - """403 Forbidden""" - - status_code: int = 403 - title: str = "Forbidden" - - -class NotFound(HTTPException): - """404 Not Found""" - - status_code: int = 404 - title: str = "Not Found" - - -class UnprocessableEntity(HTTPException): - """422 Unprocessable Entity""" - - status_code: int = 422 - title: str = "Unprocessable Entity" - - -class InternalServerError(HTTPException): - """500 Internal Server Error""" - - status_code: int = 500 - title: str = "Internal Server Error" - - -class NotImplementedResponse(HTTPException): - """501 Not Implemented""" - - status_code: int = 501 - title: str = "Not Implemented" - - -POSSIBLE_ERRORS = ( - BadRequest, - Forbidden, - NotFound, - UnprocessableEntity, - InternalServerError, - NotImplementedResponse, - VersionNotSupported, -) diff --git a/optimade/server/middleware.py b/optimade/server/middleware.py index 99196715b..ed9a69dc2 100644 --- a/optimade/server/middleware.py +++ b/optimade/server/middleware.py @@ -16,11 +16,11 @@ from starlette.requests import Request from starlette.responses import RedirectResponse, StreamingResponse +from optimade.exceptions import BadRequest, VersionNotSupported from optimade.models import Warnings from optimade.server.config import CONFIG -from optimade.server.exceptions import BadRequest, VersionNotSupported from optimade.server.routers.utils import BASE_URL_PREFIXES, get_base_url -from optimade.server.warnings import ( +from optimade.warnings import ( FieldValueNotRecognized, OptimadeWarning, QueryParamNotUsed, @@ -268,9 +268,9 @@ async def dispatch(self, request: Request, call_next): class AddWarnings(BaseHTTPMiddleware): """ - Add [`OptimadeWarning`][optimade.server.warnings.OptimadeWarning]s to the response. + Add [`OptimadeWarning`][optimade.warnings.OptimadeWarning]s to the response. - All sub-classes of [`OptimadeWarning`][optimade.server.warnings.OptimadeWarning] + All sub-classes of [`OptimadeWarning`][optimade.warnings.OptimadeWarning] will also be added to the response's [`meta.warnings`][optimade.models.optimade_json.ResponseMeta.warnings] list. @@ -332,12 +332,12 @@ def showwarning( This method will also print warning messages to `stderr` by calling `warnings._showwarning_orig()` or `warnings._showwarnmsg_impl()`. The first function will be called if the issued warning is not recognized - as an [`OptimadeWarning`][optimade.server.warnings.OptimadeWarning]. + as an [`OptimadeWarning`][optimade.warnings.OptimadeWarning]. This is equivalent to "standard behaviour". The second function will be called _after_ an - [`OptimadeWarning`][optimade.server.warnings.OptimadeWarning] has been handled. + [`OptimadeWarning`][optimade.warnings.OptimadeWarning] has been handled. - An [`OptimadeWarning`][optimade.server.warnings.OptimadeWarning] will be + An [`OptimadeWarning`][optimade.warnings.OptimadeWarning] will be translated into an OPTIMADE Warnings JSON object in accordance with [the specification](https://github.com/Materials-Consortia/OPTIMADE/blob/v1.0.0/optimade.rst#json-response-schema-common-fields). This process is similar to the [Exception handlers][optimade.server.exception_handlers]. diff --git a/optimade/server/query_params.py b/optimade/server/query_params.py index 562adf093..71cfd86b6 100644 --- a/optimade/server/query_params.py +++ b/optimade/server/query_params.py @@ -5,10 +5,10 @@ from fastapi import Query from pydantic import EmailStr # pylint: disable=no-name-in-module +from optimade.exceptions import BadRequest from optimade.server.config import CONFIG -from optimade.server.exceptions import BadRequest from optimade.server.mappers import BaseResourceMapper -from optimade.server.warnings import QueryParamNotUsed, UnknownProviderQueryParameter +from optimade.warnings import QueryParamNotUsed, UnknownProviderQueryParameter class BaseQueryParams(ABC): diff --git a/optimade/server/routers/utils.py b/optimade/server/routers/utils.py index b4003bdd0..ac19df72f 100644 --- a/optimade/server/routers/utils.py +++ b/optimade/server/routers/utils.py @@ -9,6 +9,7 @@ from starlette.datastructures import URL as StarletteURL from optimade import __api_version__ +from optimade.exceptions import BadRequest, InternalServerError from optimade.models import ( EntryResource, EntryResponseMany, @@ -18,7 +19,6 @@ ) from optimade.server.config import CONFIG from optimade.server.entry_collections import EntryCollection -from optimade.server.exceptions import BadRequest, InternalServerError from optimade.server.query_params import EntryListingQueryParams, SingleEntryQueryParams from optimade.utils import PROVIDER_LIST_URLS, get_providers, mongo_id_for_database diff --git a/optimade/server/schemas.py b/optimade/server/schemas.py index 2ac7e7485..558745548 100644 --- a/optimade/server/schemas.py +++ b/optimade/server/schemas.py @@ -22,7 +22,7 @@ submodules (e.g., the validator) to access the other schemas (that only require pydantic to construct). """ - from optimade.server.exceptions import POSSIBLE_ERRORS + from optimade.exceptions import POSSIBLE_ERRORS ERROR_RESPONSES: Optional[Dict[int, Dict]] = { err.status_code: {"model": ErrorResponse, "description": err.title} diff --git a/optimade/server/warnings.py b/optimade/server/warnings.py index 7df91c685..f8ecdb15d 100644 --- a/optimade/server/warnings.py +++ b/optimade/server/warnings.py @@ -1,67 +1,26 @@ -from typing import Optional - - -class OptimadeWarning(Warning): - """Base Warning for the `optimade` package""" - - def __init__( - self, detail: Optional[str] = None, title: Optional[str] = None, *args - ) -> None: - detail = detail if detail else self.__doc__ - super().__init__(detail, *args) - self.detail = detail - self.title = title if title else self.__class__.__name__ - - def __repr__(self) -> str: - attrs = {"detail": self.detail, "title": self.title} - return "<{:s}({:s})>".format( - self.__class__.__name__, - " ".join( - [ - f"{attr}={value!r}" - for attr, value in attrs.items() - if value is not None - ] - ), - ) - - def __str__(self) -> str: - return self.detail if self.detail is not None else "" - - -class FieldValueNotRecognized(OptimadeWarning): - """A field or value used in the request is not recognised by this implementation.""" - - -class TooManyValues(OptimadeWarning): - """A field or query parameter has too many values to be handled by this implementation.""" - - -class QueryParamNotUsed(OptimadeWarning): - """A query parameter is not used in this request.""" - - -class MissingExpectedField(OptimadeWarning): - """A field was provided with a null value when a related field was provided - with a value.""" - - -class TimestampNotRFCCompliant(OptimadeWarning): - """A timestamp has been used in a filter that contains microseconds and is thus not - RFC 3339 compliant. This may cause undefined behaviour in the query results. - - """ - - -class UnknownProviderProperty(OptimadeWarning): - """A provider-specific property has been requested via `response_fields` or as in a `filter` that is not - recognised by this implementation. - - """ - - -class UnknownProviderQueryParameter(OptimadeWarning): - """A provider-specific query parameter has been requested in the query with a prefix not - recognised by this implementation. - - """ +"""This submodule maintains backwards compatibility with the old `optimade.server.warnings` module, +which previously implemented the imported warnings directly. + +""" + +from optimade.warnings import ( + FieldValueNotRecognized, + MissingExpectedField, + OptimadeWarning, + QueryParamNotUsed, + TimestampNotRFCCompliant, + TooManyValues, + UnknownProviderProperty, + UnknownProviderQueryParameter, +) + +__all__ = ( + "FieldValueNotRecognized", + "MissingExpectedField", + "OptimadeWarning", + "QueryParamNotUsed", + "TimestampNotRFCCompliant", + "TooManyValues", + "UnknownProviderProperty", + "UnknownProviderQueryParameter", +) diff --git a/optimade/utils.py b/optimade/utils.py index c2e873897..6f4e3be78 100644 --- a/optimade/utils.py +++ b/optimade/utils.py @@ -114,7 +114,7 @@ def get_child_database_links( Raises: RuntimeError: If the provider's index meta-database is down, - invalid, or the request otherwise fails. + invalid, or the request otherwise fails. """ import requests diff --git a/optimade/warnings.py b/optimade/warnings.py new file mode 100644 index 000000000..4fea44be9 --- /dev/null +++ b/optimade/warnings.py @@ -0,0 +1,83 @@ +"""This submodule implements the possible warnings that can be omitted by an +OPTIMADE API. + +""" + +from typing import Optional + +__all__ = ( + "OptimadeWarning", + "FieldValueNotRecognized", + "TooManyValues", + "QueryParamNotUsed", + "MissingExpectedField", + "TimestampNotRFCCompliant", + "UnknownProviderProperty", + "UnknownProviderQueryParameter", +) + + +class OptimadeWarning(Warning): + """Base Warning for the `optimade` package""" + + def __init__( + self, detail: Optional[str] = None, title: Optional[str] = None, *args + ) -> None: + detail = detail if detail else self.__doc__ + super().__init__(detail, *args) + self.detail = detail + self.title = title if title else self.__class__.__name__ + + def __repr__(self) -> str: + attrs = {"detail": self.detail, "title": self.title} + return "<{:s}({:s})>".format( + self.__class__.__name__, + " ".join( + [ + f"{attr}={value!r}" + for attr, value in attrs.items() + if value is not None + ] + ), + ) + + def __str__(self) -> str: + return self.detail if self.detail is not None else "" + + +class FieldValueNotRecognized(OptimadeWarning): + """A field or value used in the request is not recognised by this implementation.""" + + +class TooManyValues(OptimadeWarning): + """A field or query parameter has too many values to be handled by this implementation.""" + + +class QueryParamNotUsed(OptimadeWarning): + """A query parameter is not used in this request.""" + + +class MissingExpectedField(OptimadeWarning): + """A field was provided with a null value when a related field was provided + with a value.""" + + +class TimestampNotRFCCompliant(OptimadeWarning): + """A timestamp has been used in a filter that contains microseconds and is thus not + RFC 3339 compliant. This may cause undefined behaviour in the query results. + + """ + + +class UnknownProviderProperty(OptimadeWarning): + """A provider-specific property has been requested via `response_fields` or as in a `filter` that is not + recognised by this implementation. + + """ + + +class UnknownProviderQueryParameter(OptimadeWarning): + """A provider-specific query parameter has been requested in the query with a prefix not + recognised by this implementation. + + """ diff --git a/requirements-server.txt b/requirements-server.txt new file mode 100644 index 000000000..b69a210be --- /dev/null +++ b/requirements-server.txt @@ -0,0 +1,5 @@ +elasticsearch==7.17.7 +elasticsearch-dsl==7.4.0 +fastapi==0.86.0 +mongomock==4.1.2 +pymongo==4.3.2 diff --git a/requirements.txt b/requirements.txt index 22498120b..9691d65c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,6 @@ -elasticsearch==7.17.7 -elasticsearch-dsl==7.4.0 email_validator==1.3.0 -fastapi==0.86.0 lark==1.1.4 -mongomock==4.1.2 pydantic==1.10.2 -pymongo==4.3.2 pyyaml==5.4 requests==2.28.1 uvicorn==0.19.0 diff --git a/tasks.py b/tasks.py index ba94966bc..216cdd753 100644 --- a/tasks.py +++ b/tasks.py @@ -231,14 +231,14 @@ def write_file(full_path: Path, content: str): full_path=docs_dir.joinpath(".pages"), content=pages_template.format(name="API Reference"), ) - continue docs_sub_dir = docs_dir.joinpath(relpath) docs_sub_dir.mkdir(exist_ok=True) - write_file( - full_path=docs_sub_dir.joinpath(".pages"), - content=pages_template.format(name=str(relpath).split("/")[-1]), - ) + if str(relpath) != ".": + write_file( + full_path=docs_sub_dir.joinpath(".pages"), + content=pages_template.format(name=str(relpath).split("/")[-1]), + ) # Create markdown files for filename in filenames: @@ -249,6 +249,10 @@ def write_file(full_path: Path, content: str): basename = filename[: -len(".py")] py_path = f"optimade/{relpath}/{basename}".replace("/", ".") + if str(relpath) == ".": + py_path = py_path.replace("...", ".") + print(filename, basename, py_path) + md_filename = filename.replace(".py", ".md") # For models we want to include EVERYTHING, even if it doesn't have a doc-string diff --git a/tests/filterparser/test_filterparser.py b/tests/filterparser/test_filterparser.py index 3c5186242..5296bf797 100644 --- a/tests/filterparser/test_filterparser.py +++ b/tests/filterparser/test_filterparser.py @@ -4,8 +4,8 @@ import pytest from lark import Tree +from optimade.exceptions import BadRequest from optimade.filterparser import LarkParser -from optimade.server.exceptions import BadRequest class BaseTestFilterParser(abc.ABC): diff --git a/tests/filtertransformers/test_elasticsearch.py b/tests/filtertransformers/test_elasticsearch.py index da70d95e6..3d076a791 100644 --- a/tests/filtertransformers/test_elasticsearch.py +++ b/tests/filtertransformers/test_elasticsearch.py @@ -1,7 +1,8 @@ import pytest elasticsearch_dsl = pytest.importorskip( - "elasticsearch_dsl", reason="No ElasticSearch installation, skipping tests..." + "elasticsearch_dsl", + reason="ElasticSearch dependencies (elasticsearch_dsl, elasticsearch) are required to run these tests.", ) from optimade.filterparser import LarkParser diff --git a/tests/filtertransformers/test_mongo.py b/tests/filtertransformers/test_mongo.py index b62ad8bbf..b4dd9cced 100644 --- a/tests/filtertransformers/test_mongo.py +++ b/tests/filtertransformers/test_mongo.py @@ -1,9 +1,15 @@ import pytest + +_ = pytest.importorskip( + "bson", + reason="MongoDB dependency set (pymongo, bson) are required to run these tests.", +) + from lark.exceptions import VisitError +from optimade.exceptions import BadRequest from optimade.filterparser import LarkParser -from optimade.server.exceptions import BadRequest -from optimade.server.warnings import UnknownProviderProperty +from optimade.warnings import UnknownProviderProperty class TestMongoTransformer: @@ -526,7 +532,7 @@ def test_suspected_timestamp_fields(self, mapper): import bson.tz_util from optimade.filtertransformers.mongo import MongoTransformer - from optimade.server.warnings import TimestampNotRFCCompliant + from optimade.warnings import TimestampNotRFCCompliant example_RFC3339_date = "2019-06-08T04:13:37Z" example_RFC3339_date_2 = "2019-06-08T04:13:37" diff --git a/tests/models/conftest.py b/tests/models/conftest.py index ff0daebee..a6cc48564 100644 --- a/tests/models/conftest.py +++ b/tests/models/conftest.py @@ -49,6 +49,15 @@ def good_structures() -> list: return structures +@pytest.fixture(scope="session") +def good_references() -> list: + """Load and return list of good structures resources""" + filename = "test_good_references.json" + references = load_test_data(filename) + references = remove_mongo_date(references) + return references + + @pytest.fixture def starting_links() -> dict: """A good starting links resource""" diff --git a/tests/models/test_data/test_good_references.json b/tests/models/test_data/test_good_references.json new file mode 100644 index 000000000..bf55373be --- /dev/null +++ b/tests/models/test_data/test_good_references.json @@ -0,0 +1,50 @@ +[ + { + "id": "dijkstra1968", + "type": "references", + "last_modified": "2019-11-12T14:24:37.331000", + "authors": [ + { + "name": "Edsger W. Dijkstra", + "firstname": "Edsger", + "lastname": "Dijkstra" + } + ], + "doi": "10.1145/362929.362947", + "journal": "Communications of the ACM", + "title": "Go To Statement Considered Harmful", + "year": "1968" + }, + { + "id": "maddox1988", + "type": "references", + "last_modified": "2019-11-27T14:24:37.331000", + "authors": [ + { + "name": "John Maddox", + "firstname": "John", + "lastname": "Maddox" + } + ], + "doi": "10.1038/335201a0", + "journal": "Nature", + "title": "Crystals From First Principles", + "year": "1988" + }, + { + "id": "dummy/2019", + "type": "references", + "last_modified": "2019-11-23T14:24:37.332000", + "authors": [ + { + "name": "A Nother", + "firstname": "A", + "lastname": "Nother" + } + ], + "doi": "10.1038/00000", + "journal": "JACS", + "title": "Dummy reference that should remain orphaned from all structures for testing purposes", + "year": "2019" + } +] diff --git a/tests/models/test_links.py b/tests/models/test_links.py index c6d011117..c65cc9edf 100644 --- a/tests/models/test_links.py +++ b/tests/models/test_links.py @@ -8,12 +8,6 @@ def test_good_links(starting_links, mapper): """Check well-formed links used as example data""" - import optimade.server.data - - good_refs = optimade.server.data.links - for doc in good_refs: - LinksResource(**mapper(MAPPER).map_back(doc)) - # Test starting_links is a good links resource LinksResource(**mapper(MAPPER).map_back(starting_links)) diff --git a/tests/models/test_references.py b/tests/models/test_references.py index ecab38dff..de86bebb4 100644 --- a/tests/models/test_references.py +++ b/tests/models/test_references.py @@ -1,18 +1,23 @@ # pylint: disable=no-member import pytest +from pydantic import ValidationError from optimade.models.references import ReferenceResource MAPPER = "ReferenceMapper" -def test_good_references(mapper): - """Check well-formed references used as example data""" - import optimade.server.data - - good_refs = optimade.server.data.references - for doc in good_refs: - ReferenceResource(**mapper(MAPPER).map_back(doc)) +def test_more_good_references(good_references, mapper): + """Check well-formed structures with specific edge-cases""" + for index, structure in enumerate(good_references): + try: + ReferenceResource(**mapper(MAPPER).map_back(structure)) + except ValidationError: + # Printing to keep the original exception as is, while still being informational + print( + f"Good test structure {index} failed to validate from 'test_good_structures.json'" + ) + raise def test_bad_references(mapper): diff --git a/tests/models/test_structures.py b/tests/models/test_structures.py index a11e00b56..60f811563 100644 --- a/tests/models/test_structures.py +++ b/tests/models/test_structures.py @@ -5,21 +5,11 @@ from pydantic import ValidationError from optimade.models.structures import CORRELATED_STRUCTURE_FIELDS, StructureResource -from optimade.server.warnings import MissingExpectedField +from optimade.warnings import MissingExpectedField MAPPER = "StructureMapper" -def test_good_structures(mapper): - """Check well-formed structures used as example data""" - import optimade.server.data - - good_structures = optimade.server.data.structures - - for structure in good_structures: - StructureResource(**mapper(MAPPER).map_back(structure)) - - @pytest.mark.filterwarnings("ignore", category=MissingExpectedField) def test_good_structure_with_missing_data(mapper, good_structure): """Check deserialization of well-formed structure used diff --git a/tests/server/conftest.py b/tests/server/conftest.py index de45cd578..175d3f8fa 100644 --- a/tests/server/conftest.py +++ b/tests/server/conftest.py @@ -2,7 +2,7 @@ import pytest -from optimade.server.warnings import OptimadeWarning +from optimade.warnings import OptimadeWarning @pytest.fixture(scope="session") diff --git a/tests/server/middleware/test_query_param.py b/tests/server/middleware/test_query_param.py index 06ca9ce9c..03998a2dc 100644 --- a/tests/server/middleware/test_query_param.py +++ b/tests/server/middleware/test_query_param.py @@ -1,9 +1,9 @@ """Test EntryListingQueryParams middleware""" import pytest -from optimade.server.exceptions import BadRequest +from optimade.exceptions import BadRequest from optimade.server.middleware import EnsureQueryParamIntegrity -from optimade.server.warnings import FieldValueNotRecognized +from optimade.warnings import FieldValueNotRecognized def test_wrong_html_form(check_error_response, both_clients): diff --git a/tests/server/test_client.py b/tests/server/test_client.py index ea1a5c02c..89846e02f 100644 --- a/tests/server/test_client.py +++ b/tests/server/test_client.py @@ -4,7 +4,7 @@ import pytest -from optimade.server.warnings import MissingExpectedField +from optimade.warnings import MissingExpectedField try: from optimade.client import OptimadeClient