From ba9b46f155937b43eb265fa0b7c9ad34c9309850 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Fri, 13 Mar 2020 13:16:06 +0000 Subject: [PATCH 1/8] Update invoke task to scrape optional filters --- optimade/validator/data/filters.txt | 12 ------------ optimade/validator/data/optional_filters.txt | 7 +++++-- tasks.py | 14 +++++++++++++- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/optimade/validator/data/filters.txt b/optimade/validator/data/filters.txt index 3092fe1d6..06473bf17 100644 --- a/optimade/validator/data/filters.txt +++ b/optimade/validator/data/filters.txt @@ -9,8 +9,6 @@ _exmpl_bandgap > 5.0 AND _exmpl_molecular_weight < 350 _exmpl_melting_point<300 AND nelements=4 AND elements="Si,O2" _exmpl_some_string_property = 42 5 < _exmpl_a -((NOT (_exmpl_a>_exmpl_b)) AND _exmpl_x>0) -5 < 7 identifier CONTAINS x identifier STARTS WITH x identifier ENDS WITH x @@ -20,16 +18,7 @@ list HAS value list HAS ALL values list HAS ANY values list LENGTH value -list HAS ONLY values NOT list HAS inverse -list HAS < 3 -list HAS ALL < 3, > 3 -list:list HAS >=2:<=5 -elements HAS "H" AND elements HAS ALL "H","He","Ga","Ta" AND elements HAS ONLY "H","He","Ga","Ta" AND elements HAS ANY "H", "He", "Ga", "Ta" -elements HAS ONLY "H","He","Ga","Ta" -elements:_exmpl_element_counts HAS "H":6 AND elements:_exmpl_element_counts HAS ALL "H":6,"He":7 AND elements:_exmpl_element_counts HAS ONLY "H":6 AND elements:_exmpl_element_counts HAS ANY "H":6,"He":7 AND elements:_exmpl_element_counts HAS ONLY "H":6,"He":7 -_exmpl_element_counts HAS < 3 AND _exmpl_element_counts HAS ANY > 3, = 6, 4, != 8 -elements:_exmpl_element_counts:_exmpl_element_weights HAS ANY > 3:"He":>55.3 , = 6:>"Ti":<37.6 , 8:<"Ga":0 calculations.id HAS "calc-id-96" authors.lastname HAS "Schmit" identifier IS UNKNOWN @@ -43,7 +32,6 @@ elements HAS ALL "Si", "Al", "O" elements HAS ALL "Si", "Al", "O" AND elements LENGTH 3 nelements=4 nelements>=2 AND nelements<=7 -elements:elements_ratios HAS ALL "Al":>0.3333, "Al":<0.3334 chemical_formula_descriptive="(H2O)2 Na" chemical_formula_descriptive CONTAINS "H2O" chemical_formula_reduced="H2NaO" diff --git a/optimade/validator/data/optional_filters.txt b/optimade/validator/data/optional_filters.txt index 7de1d1451..8a02d12fc 100644 --- a/optimade/validator/data/optional_filters.txt +++ b/optimade/validator/data/optional_filters.txt @@ -1,9 +1,12 @@ +((NOT (_exmpl_a>_exmpl_b)) AND _exmpl_x>0) +5 < 7 list HAS ONLY values list HAS < 3 list HAS ALL < 3, > 3 list:list HAS >=2:<=5 -elements:elements_ratios HAS ALL "Al":>0.3333, "Al":<0.3334 +elements HAS "H" AND elements HAS ALL "H","He","Ga","Ta" AND elements HAS ONLY "H","He","Ga","Ta" AND elements HAS ANY "H", "He", "Ga", "Ta" +elements HAS ONLY "H","He","Ga","Ta" elements:_exmpl_element_counts HAS "H":6 AND elements:_exmpl_element_counts HAS ALL "H":6,"He":7 AND elements:_exmpl_element_counts HAS ONLY "H":6 AND elements:_exmpl_element_counts HAS ANY "H":6,"He":7 AND elements:_exmpl_element_counts HAS ONLY "H":6,"He":7 _exmpl_element_counts HAS < 3 AND _exmpl_element_counts HAS ANY > 3, = 6, 4, != 8 elements:_exmpl_element_counts:_exmpl_element_weights HAS ANY > 3:"He":>55.3 , = 6:>"Ti":<37.6 , 8:<"Ga":0 -5 < 7 +elements:elements_ratios HAS ALL "Al":>0.3333, "Al":<0.3334 diff --git a/tasks.py b/tasks.py index 6558e1f8b..9e9bcb10f 100644 --- a/tasks.py +++ b/tasks.py @@ -113,6 +113,9 @@ def parse_spec_for_filters(_): import requests filter_path = TOP_DIR.joinpath("optimade/validator/data/filters.txt") + optional_filter_path = TOP_DIR.joinpath( + "optimade/validator/data/optional_filters.txt" + ) specification_flines = ( requests.get( @@ -123,12 +126,21 @@ def parse_spec_for_filters(_): ) filters = [] + optional_filters = [] for line in specification_flines: if ":filter:" in line: + print(line) for _split in line.replace("filter=", "").split(":filter:")[1:]: _filter = _split.split("`")[1].strip() - filters.append(_filter) + if "OPTIONAL" in line: + optional_filters.append(_filter) + else: + filters.append(_filter) with open(filter_path, "w") as f: for _filter in filters: f.write(_filter + "\n") + + with open(optional_filter_path, "w") as f: + for _filter in optional_filters: + f.write(_filter + "\n") From b592f54995544fba31f31a9c8c5584feadaf5c3c Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Fri, 13 Mar 2020 13:20:15 +0000 Subject: [PATCH 2/8] Load and expose optional filters --- optimade/validator/data/__init__.py | 52 ++++++++++++++++++----------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/optimade/validator/data/__init__.py b/optimade/validator/data/__init__.py index 79c9c7a2f..73503ff57 100644 --- a/optimade/validator/data/__init__.py +++ b/optimade/validator/data/__init__.py @@ -1,13 +1,9 @@ """ This submodule loads the filter examples that have been scraped from the specification. -- `filters.txt` is a automatically generated file containing all -examples. It is generated using the `invoke` task `parse_spec_for_filters`. -- `optional_filters.txt` is human-generated, and allows for filters that -are inside `filters.txt` to be skipped, if it has been deemed that they -are optional (external to the parsed file). In the future, it is hoped -that the spec will be fully parseable and as such this file could be -generated automatically too. +- Both `filters.txt` and `optional_filters.txt.` are automatically +generated files containing all examples from the specification. It +is generated using the `invoke` task `parse_spec_for_filters`. When being loaded in, these filter examples are also made more concrete by replacing references to vague `values`/`value` with specific examples. @@ -16,21 +12,37 @@ from pathlib import Path -__all__ = ["MANDATORY_FILTER_EXAMPLES"] +__all__ = ["MANDATORY_FILTER_EXAMPLES", "OPTIONAL_FILTER_EXAMPLES"] ALIASES = {"values": '"1", "2", "3"', "value": "1", "inverse": "1"} -with open(Path(__file__).parent.joinpath("optional_filters.txt"), "r") as f: - OPTIONAL_FILTER_EXAMPLES = [line.strip() for line in f.readlines()] -with open(Path(__file__).parent.joinpath("filters.txt"), "r") as f: - _filters = [] - for line in f.readlines(): - if line.strip() in OPTIONAL_FILTER_EXAMPLES: - continue - for alias in ALIASES: - if alias in line: - line = line.replace(alias, ALIASES[alias]) - _filters.append(line) +def _load_filters_and_apply_aliases(path): + """ Load a text file containing example filters with one + filter per line, and apply aliases to swap out dummy values + with more concrete examples. - MANDATORY_FILTER_EXAMPLES = _filters + Parameters: + path (str/Path): the filename to load. + + Returns: + list: a list of filters. + + """ + with open(path, "r") as f: + _filters = [] + for line in f.readlines(): + for alias in ALIASES: + if alias in line: + line = line.replace(alias, ALIASES[alias]) + _filters.append(line) + + return _filters + + +OPTIONAL_FILTER_EXAMPLES = _load_filters_and_apply_aliases( + Path(__file__).parent.joinpath("optional_filters.txt") +) +MANDATORY_FILTER_EXAMPLES = _load_filters_and_apply_aliases( + Path(__file__).parent.joinpath("filters.txt") +) From 9597729add48d29c315c00e6709482fa06be9e5b Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Fri, 13 Mar 2020 14:22:19 +0000 Subject: [PATCH 3/8] Added optional filter validation to validator --- optimade/validator/validator.py | 88 ++++++++++++++++++++++++++++----- 1 file changed, 76 insertions(+), 12 deletions(-) diff --git a/optimade/validator/validator.py b/optimade/validator/validator.py index 53c383750..17cf933dd 100644 --- a/optimade/validator/validator.py +++ b/optimade/validator/validator.py @@ -18,7 +18,7 @@ from optimade.models import InfoResponse, EntryInfoResponse, IndexInfoResponse -from .data import MANDATORY_FILTER_EXAMPLES +from .data import MANDATORY_FILTER_EXAMPLES, OPTIONAL_FILTER_EXAMPLES from .validator_model_patches import ( ValidatorLinksResponse, ValidatorEntryResponseOne, @@ -39,6 +39,11 @@ "references": [], } +ENDPOINT_OPTIONAL_QUERIES = { + "structures": OPTIONAL_FILTER_EXAMPLES, + "references": [], +} + RESPONSE_CLASSES = { "references": ValidatorReferenceResponseMany, "references/": ValidatorReferenceResponseOne, @@ -148,11 +153,14 @@ def test_case(test_fn): and a message to print upon success. Should raise `ResponseError`, `ValidationError` or `ManualValidationError` if the test case has failed. + Keyword arguments: + optional (bool): whether or not to treat the test as optional. + """ from functools import wraps @wraps(test_fn) - def wrapper(validator, *args, **kwargs): + def wrapper(validator, *args, optional=False, **kwargs): try: result, msg = test_fn(validator, *args, **kwargs) except json.JSONDecodeError as exc: @@ -169,24 +177,46 @@ def wrapper(validator, *args, **kwargs): request = validator.client.last_request except AttributeError: request = validator.base_url + if result is not None: - validator.success_count += 1 + if not optional: + validator.success_count += 1 + else: + validator.optional_success_count += 1 + message = f"✔: {request} - {msg}" if validator.verbosity > 0: - print_success(f"✔: {request} - {msg}") + if optional: + print(message) + else: + print_success(message) else: - print_success(".", end="", flush=True) + if optional: + print(".", end="", flush=True) + else: + print_success(".", end="", flush=True) else: - validator.failure_count += 1 + if not optional: + validator.failure_count += 1 + else: + validator.optional_failure_count += 1 request = request.replace("\n", "") message = f"{msg}".split("\n") summary = f"✖: {request} - {test_fn.__name__} - failed with error" validator.failure_messages.append((summary, message)) if validator.verbosity > 0: - print_failure(summary) - for line in message: - print_warning(f"\t{line}") + if optional: + print(summary) + for line in message: + print(f"\t{line}") + else: + print_failure(summary) + for line in message: + print_warning(f"\t{line}") else: - print_failure(f"✖", end="", flush=True) + if optional: + print(f"✖", end="", flush=True) + else: + print_failure(f"✖", end="", flush=True) return result @@ -262,6 +292,8 @@ def __init__( # pylint: disable=too-many-arguments {} if self.index else ENDPOINT_MANDATORY_QUERIES ) + self.endpoint_optional_queries = {} if self.index else ENDPOINT_OPTIONAL_QUERIES + self.response_classes = ( RESPONSE_CLASSES_INDEX if self.index else RESPONSE_CLASSES ) @@ -273,6 +305,8 @@ def __init__( # pylint: disable=too-many-arguments self.success_count = 0 self.failure_count = 0 + self.optional_success_count = 0 + self.optional_failure_count = 0 self.failure_messages = [] def _setup_log(self): @@ -315,6 +349,7 @@ def main(self): # test entire implementation print(f"Testing entire implementation at {self.base_url}...") + print("\nMandatory tests:") self._log.debug("Testing base info endpoint of %s", BASE_INFO_ENDPOINT) base_info = self.test_info_or_links_endpoints(BASE_INFO_ENDPOINT) self.get_available_endpoints(base_info) @@ -345,6 +380,15 @@ def main(self): self.valid = not bool(self.failure_count) + print("\nOptional tests:") + for endp in self.endpoint_optional_queries: + # skip empty endpoint query lists + if self.endpoint_mandatory_queries[endp]: + self._log.debug("Testing optional query syntax on endpoint %s", endp) + self.test_optional_query_syntax( + endp, self.endpoint_optional_queries[endp] + ) + if not self.valid: print(f"\n\nFAILURES\n") for message in self.failure_messages: @@ -352,12 +396,17 @@ def main(self): for line in message[1]: print_warning("\t" + line) - final_message = f"\nPassed {self.success_count} out of {self.success_count + self.failure_count} tests." + final_message = f"\n\nPassed {self.success_count} out of {self.success_count + self.failure_count} tests." if not self.valid: print_failure(final_message) else: print_success(final_message) + print( + f"Additionally passed {self.optional_success_count} out of " + f"{self.optional_success_count + self.optional_failure_count} optional tests." + ) + def test_info_or_links_endpoints(self, request_str): """ Runs the test cases for the info endpoints. """ response = self.get_endpoint(request_str) @@ -512,7 +561,7 @@ def get_available_endpoints(self, base_info): ) @test_case - def get_endpoint(self, request_str): + def get_endpoint(self, request_str, optional=False): """ Gets the response from the endpoint specified by `request_str`. """ request_str = f"/{request_str}".replace("\n", "") response = self.client.get(request_str) @@ -544,3 +593,18 @@ def test_mandatory_query_syntax(self, endpoint, endpoint_queries): valid_queries = [f"{endpoint}?filter={query}" for query in endpoint_queries] for query in valid_queries: self.get_endpoint(query) + + def test_optional_query_syntax(self, endpoint, endpoint_queries): + """ Perform a list of valid queries and assert that no errors are raised. + + Parameters: + endpoint (str): the endpoint to query (e.g. "structures"). + endpoint_queries (list): the list of valid mandatory queries + for that endpoint, where the queries do not include the + "?filter=" prefix, e.g. ['elements HAS "Na"']. + + """ + + valid_queries = [f"{endpoint}?filter={query}" for query in endpoint_queries] + for query in valid_queries: + self.get_endpoint(query, optional=True) From 7cd7742d7d3be28cce5d5ca5dfa43b83ed8e4772 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Fri, 13 Mar 2020 14:30:51 +0000 Subject: [PATCH 4/8] Ignore optional validation failures in test suite --- tests/server/test_server_validation.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/server/test_server_validation.py b/tests/server/test_server_validation.py index e7e1ce956..d4beb35e1 100644 --- a/tests/server/test_server_validation.py +++ b/tests/server/test_server_validation.py @@ -1,6 +1,7 @@ # pylint: disable=relative-beyond-top-level,import-outside-toplevel import os import unittest +from traceback import print_exc from optimade.validator import ImplementationValidator @@ -13,7 +14,10 @@ class ServerTestWithValidator(SetClient, unittest.TestCase): def test_with_validator(self): validator = ImplementationValidator(client=self.client) - validator.main() + try: + validator.main() + except Exception: + print_exc() self.assertTrue(validator.valid) @@ -23,7 +27,10 @@ class IndexServerTestWithValidator(SetClient, unittest.TestCase): def test_with_validator(self): validator = ImplementationValidator(client=self.client, index=True) - validator.main() + try: + validator.main() + except Exception: + print_exc() self.assertTrue(validator.valid) From d4abe5e3d0ef6c4fe464c0dc9d0439ded442f5b7 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Tue, 17 Mar 2020 12:55:45 +0000 Subject: [PATCH 5/8] Resolve paths when loading filters Co-authored-by: Casper Welzel Andersen --- optimade/validator/data/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/optimade/validator/data/__init__.py b/optimade/validator/data/__init__.py index 73503ff57..501d487a2 100644 --- a/optimade/validator/data/__init__.py +++ b/optimade/validator/data/__init__.py @@ -41,8 +41,8 @@ def _load_filters_and_apply_aliases(path): OPTIONAL_FILTER_EXAMPLES = _load_filters_and_apply_aliases( - Path(__file__).parent.joinpath("optional_filters.txt") + Path(__file__).parent.joinpath("optional_filters.txt").resolve() ) MANDATORY_FILTER_EXAMPLES = _load_filters_and_apply_aliases( - Path(__file__).parent.joinpath("filters.txt") + Path(__file__).parent.joinpath("filters.txt").resolve() ) From b77c05b5093f05b03c95ccdd941549c65c82ddb5 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Tue, 17 Mar 2020 12:58:15 +0000 Subject: [PATCH 6/8] Removed stray print statement Co-authored-by: Casper Welzel Andersen --- tasks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tasks.py b/tasks.py index 9e9bcb10f..46195e643 100644 --- a/tasks.py +++ b/tasks.py @@ -129,7 +129,6 @@ def parse_spec_for_filters(_): optional_filters = [] for line in specification_flines: if ":filter:" in line: - print(line) for _split in line.replace("filter=", "").split(":filter:")[1:]: _filter = _split.split("`")[1].strip() if "OPTIONAL" in line: From 8a7b92683d1ffa718636c915715197482d8b36a2 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Tue, 17 Mar 2020 13:01:19 +0000 Subject: [PATCH 7/8] Combine mandatory/optional query tests into one fn Co-authored-by: Casper Welzel Andersen --- optimade/validator/validator.py | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/optimade/validator/validator.py b/optimade/validator/validator.py index 17cf933dd..67956a625 100644 --- a/optimade/validator/validator.py +++ b/optimade/validator/validator.py @@ -371,9 +371,7 @@ def main(self): # skip empty endpoint query lists if self.endpoint_mandatory_queries[endp]: self._log.debug("Testing mandatory query syntax on endpoint %s", endp) - self.test_mandatory_query_syntax( - endp, self.endpoint_mandatory_queries[endp] - ) + self.test_query_syntax(endp, self.endpoint_mandatory_queries[endp]) self._log.debug("Testing %s endpoint", LINKS_ENDPOINT) self.test_info_or_links_endpoints(LINKS_ENDPOINT) @@ -385,8 +383,8 @@ def main(self): # skip empty endpoint query lists if self.endpoint_mandatory_queries[endp]: self._log.debug("Testing optional query syntax on endpoint %s", endp) - self.test_optional_query_syntax( - endp, self.endpoint_optional_queries[endp] + self.test_query_syntax( + endp, self.endpoint_optional_queries[endp], optional=True ) if not self.valid: @@ -579,8 +577,9 @@ def get_endpoint(self, request_str, optional=False): return response, "request successful." - def test_mandatory_query_syntax(self, endpoint, endpoint_queries): - """ Perform a list of valid queries and assert that no errors are raised. + def test_query_syntax(self, endpoint, endpoint_queries, optional=False): + """ Execute a list of valid queries agains the endpoint and assert + that no errors are raised. Parameters: endpoint (str): the endpoint to query (e.g. "structures"). @@ -588,23 +587,11 @@ def test_mandatory_query_syntax(self, endpoint, endpoint_queries): for that endpoint, where the queries do not include the "?filter=" prefix, e.g. ['elements HAS "Na"']. - """ - - valid_queries = [f"{endpoint}?filter={query}" for query in endpoint_queries] - for query in valid_queries: - self.get_endpoint(query) - - def test_optional_query_syntax(self, endpoint, endpoint_queries): - """ Perform a list of valid queries and assert that no errors are raised. - - Parameters: - endpoint (str): the endpoint to query (e.g. "structures"). - endpoint_queries (list): the list of valid mandatory queries - for that endpoint, where the queries do not include the - "?filter=" prefix, e.g. ['elements HAS "Na"']. + Keyword arguments: + optional (bool): treat the success of the queries as optional. """ valid_queries = [f"{endpoint}?filter={query}" for query in endpoint_queries] for query in valid_queries: - self.get_endpoint(query, optional=True) + self.get_endpoint(query, optional=optional) From 6715bfcc6b107b275309105c79a262244b580eed Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Tue, 17 Mar 2020 13:03:30 +0000 Subject: [PATCH 8/8] Created tuple of triggers for optionality in spec Co-authored-by: Casper Welzel Andersen --- tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tasks.py b/tasks.py index 46195e643..cd7cada81 100644 --- a/tasks.py +++ b/tasks.py @@ -127,11 +127,12 @@ def parse_spec_for_filters(_): filters = [] optional_filters = [] + optional_triggers = ("OPTIONAL",) for line in specification_flines: if ":filter:" in line: for _split in line.replace("filter=", "").split(":filter:")[1:]: _filter = _split.split("`")[1].strip() - if "OPTIONAL" in line: + if any(trigger in line for trigger in optional_triggers): optional_filters.append(_filter) else: filters.append(_filter)