-
Notifications
You must be signed in to change notification settings - Fork 564
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Rework turtle-like test suites (#1986)
This patch reworks the turtle-like test suites (N-Quads, N-Triples,
Turtle, TriG). The changes includes:
- A new DAWG Manifest processor:
- Incorporates pytest expected failure support in test parameter generation.
- Supports URI mapping so that local filesystem URIs can be mapped to the
correct remote base URIs and still work correctly with local files.
- Supports custom entry types so that suite specific handling can be
implemented separately from the core manifest processor. This will
be used for SPARQL test suite and JSON test suite processing.
- Updated EARL reporter:
- Support for the new DAWG Manifest processor.
- Writing of EARL reports by default if a report prefix is
specified in the DAWG Manifest processor.
- Reports output is sorted so that it can be compared with text based
diff tools.
- Fixed reporting of xfail tests. These were reported as skipped before,
now they are reported as failed.
The testing is also more accurate, and now tests which were reported as
passing before are now correctly reported as failing. With one exception
all these failures are related to leniency in the parser, as our parser
accepts input that should raise an error when parsing.
Other changes:
- Moved IRI related test utilities to a separate module.
- Added generated reports for turtle-like formats to git with `-HEAD`
suffix.
Subsequent changes will use this new processor for SPARQL and RDF/XML.- Loading branch information
Showing
16 changed files
with
7,631 additions
and
401 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,42 +1,92 @@ | ||
| """This runs the nquads tests for the W3C RDF Working Group's N-Quads | ||
| test suite.""" | ||
|
|
||
|
|
||
| import os | ||
| import logging | ||
| from contextlib import ExitStack | ||
| from test.data import TEST_DATA_DIR | ||
| from test.utils.manifest import RDFTest, read_manifest | ||
| from test.utils import BNodeHandling, GraphHelper, ensure_suffix | ||
| from test.utils.dawg_manifest import ManifestEntry, params_from_sources | ||
| from test.utils.iri import URIMapper | ||
| from test.utils.namespace import RDFT | ||
| from typing import Callable, Dict | ||
| from typing import Optional | ||
|
|
||
| import pytest | ||
|
|
||
| from rdflib import ConjunctiveGraph | ||
| from rdflib.term import Node, URIRef | ||
| from rdflib.graph import Dataset | ||
|
|
||
| verbose = False | ||
| logger = logging.getLogger(__name__) | ||
|
|
||
| REMOTE_BASE_IRI = "http://www.w3.org/2013/NQuadsTests/" | ||
| LOCAL_BASE_DIR = TEST_DATA_DIR / "suites/w3c/nquads/" | ||
| ENCODING = "utf-8" | ||
| MAPPER = URIMapper.from_mappings( | ||
| (REMOTE_BASE_IRI, ensure_suffix(LOCAL_BASE_DIR.as_uri(), "/")) | ||
| ) | ||
| VALID_TYPES = {RDFT.TestNQuadsPositiveSyntax, RDFT.TestNQuadsNegativeSyntax} | ||
|
|
||
|
|
||
| def nquads(test): | ||
| g = ConjunctiveGraph() | ||
| def check_entry(entry: ManifestEntry) -> None: | ||
| assert entry.action is not None | ||
| assert entry.type in VALID_TYPES | ||
| action_path = entry.uri_mapper.to_local_path(entry.action) | ||
| if logger.isEnabledFor(logging.DEBUG): | ||
| logger.debug( | ||
| "action = %s\n%s", action_path, action_path.read_text(encoding=ENCODING) | ||
| ) | ||
| catcher: Optional[pytest.ExceptionInfo[Exception]] = None | ||
| dataset = Dataset() | ||
| with ExitStack() as xstack: | ||
| if entry.type == RDFT.TestNQuadsNegativeSyntax: | ||
| catcher = xstack.enter_context(pytest.raises(Exception)) | ||
| dataset.parse(action_path, publicID=entry.action, format="nquads") | ||
| if catcher is not None: | ||
| assert catcher.value is not None | ||
|
|
||
| try: | ||
| g.parse(test.action, format="nquads") | ||
| if not test.syntax: | ||
| raise AssertionError("Input shouldn't have parsed!") | ||
| except: | ||
| if test.syntax: | ||
| raise | ||
| if entry.type == RDFT.TestNQuadsPositiveSyntax: | ||
| graph_data = dataset.serialize(format="nquads") | ||
| result_dataset = Dataset() | ||
| result_dataset.parse(data=graph_data, publicID=entry.action, format="nquads") | ||
| GraphHelper.assert_cgraph_isomorphic( | ||
| dataset, result_dataset, exclude_bnodes=True | ||
| ) | ||
| GraphHelper.assert_sets_equals( | ||
| dataset, result_dataset, bnode_handling=BNodeHandling.COLLAPSE | ||
| ) | ||
|
|
||
|
|
||
| testers: Dict[Node, Callable[[RDFTest], None]] = { | ||
| RDFT.TestNQuadsPositiveSyntax: nquads, | ||
| RDFT.TestNQuadsNegativeSyntax: nquads, | ||
| MARK_DICT = { | ||
| f"{REMOTE_BASE_IRI}#nt-syntax-bad-uri-02": pytest.mark.xfail( | ||
| reason="accepts an invalid IRI" | ||
| ), | ||
| f"{REMOTE_BASE_IRI}#nt-syntax-bad-uri-03": pytest.mark.xfail( | ||
| reason="accepts an invalid IRI" | ||
| ), | ||
| f"{REMOTE_BASE_IRI}#nt-syntax-bad-uri-04": pytest.mark.xfail( | ||
| reason="accepts an invalid IRI" | ||
| ), | ||
| f"{REMOTE_BASE_IRI}#nt-syntax-bad-uri-05": pytest.mark.xfail( | ||
| reason="accepts an invalid IRI" | ||
| ), | ||
| f"{REMOTE_BASE_IRI}#nt-syntax-bad-esc-01": pytest.mark.xfail( | ||
| reason="accepts badly escaped literal" | ||
| ), | ||
| f"{REMOTE_BASE_IRI}#nt-syntax-bad-esc-02": pytest.mark.xfail( | ||
| reason="accepts badly escaped literal" | ||
| ), | ||
| f"{REMOTE_BASE_IRI}#nt-syntax-bad-esc-03": pytest.mark.skip( | ||
| reason="accepts badly escaped literal" | ||
| ), | ||
| } | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| "rdf_test_uri, type, rdf_test", | ||
| read_manifest(os.path.join(TEST_DATA_DIR, "suites", "w3c/nquads/manifest.ttl")), | ||
| ["manifest_entry"], | ||
| params_from_sources( | ||
| MAPPER, | ||
| ManifestEntry, | ||
| LOCAL_BASE_DIR / "manifest.ttl", | ||
| mark_dict=MARK_DICT, | ||
| report_prefix="rdflib_w3c_nquads", | ||
| ), | ||
| ) | ||
| def test_manifest(rdf_test_uri: URIRef, type: Node, rdf_test: RDFTest): | ||
| testers[type](rdf_test) | ||
| def test_entry(manifest_entry: ManifestEntry) -> None: | ||
| check_entry(manifest_entry) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,42 +1,96 @@ | ||
| """This runs the nt tests for the W3C RDF Working Group's N-Quads | ||
| """This runs the nt tests for the W3C RDF Working Group's N-Triples | ||
| test suite.""" | ||
| import os | ||
| import logging | ||
| from contextlib import ExitStack | ||
| from test.data import TEST_DATA_DIR | ||
| from test.utils.manifest import RDFTest, read_manifest | ||
| from test.utils import BNodeHandling, GraphHelper, ensure_suffix | ||
| from test.utils.dawg_manifest import ManifestEntry, params_from_sources | ||
| from test.utils.iri import URIMapper | ||
| from test.utils.namespace import RDFT | ||
| from typing import Callable, Dict | ||
| from typing import Optional | ||
|
|
||
| import pytest | ||
|
|
||
| from rdflib import Graph | ||
| from rdflib.term import Node, URIRef | ||
| from rdflib.graph import Graph | ||
|
|
||
| verbose = False | ||
| logger = logging.getLogger(__name__) | ||
|
|
||
| REMOTE_BASE_IRI = "http://www.w3.org/2013/N-TriplesTests/" | ||
| LOCAL_BASE_DIR = TEST_DATA_DIR / "suites/w3c/ntriples/" | ||
| ENCODING = "utf-8" | ||
| MAPPER = URIMapper.from_mappings( | ||
| (REMOTE_BASE_IRI, ensure_suffix(LOCAL_BASE_DIR.as_uri(), "/")) | ||
| ) | ||
| VALID_TYPES = {RDFT.TestNTriplesPositiveSyntax, RDFT.TestNTriplesNegativeSyntax} | ||
|
|
||
|
|
||
| def nt(test): | ||
| g = Graph() | ||
| def check_entry(entry: ManifestEntry) -> None: | ||
| assert entry.action is not None | ||
| assert entry.type in VALID_TYPES | ||
| action_path = entry.uri_mapper.to_local_path(entry.action) | ||
| if logger.isEnabledFor(logging.DEBUG): | ||
| logger.debug( | ||
| "action = %s\n%s", action_path, action_path.read_text(encoding=ENCODING) | ||
| ) | ||
| catcher: Optional[pytest.ExceptionInfo[Exception]] = None | ||
| graph = Graph() | ||
| with ExitStack() as xstack: | ||
| if entry.type == RDFT.TestNTriplesNegativeSyntax: | ||
| catcher = xstack.enter_context(pytest.raises(Exception)) | ||
| graph.parse(action_path, publicID=entry.action, format="ntriples") | ||
| if catcher is not None: | ||
| assert catcher.value is not None | ||
|
|
||
| try: | ||
| g.parse(test.action, format="nt") | ||
| if not test.syntax: | ||
| raise AssertionError("Input shouldn't have parsed!") | ||
| except: | ||
| if test.syntax: | ||
| raise | ||
| if entry.type == RDFT.TestNTriplesPositiveSyntax: | ||
| graph_data = graph.serialize(format="ntriples") | ||
| result_graph = Graph() | ||
| result_graph.parse(data=graph_data, publicID=entry.action, format="ntriples") | ||
| GraphHelper.assert_isomorphic(graph, result_graph) | ||
| GraphHelper.assert_sets_equals( | ||
| graph, result_graph, bnode_handling=BNodeHandling.COLLAPSE | ||
| ) | ||
|
|
||
|
|
||
| testers: Dict[Node, Callable[[RDFTest], None]] = { | ||
| RDFT.TestNTriplesPositiveSyntax: nt, | ||
| RDFT.TestNTriplesNegativeSyntax: nt, | ||
| MARK_DICT = { | ||
| f"{REMOTE_BASE_IRI}#nt-syntax-bad-uri-02": pytest.mark.xfail( | ||
| reason="accepts an invalid IRI" | ||
| ), | ||
| f"{REMOTE_BASE_IRI}#nt-syntax-bad-uri-03": pytest.mark.xfail( | ||
| reason="accepts an invalid IRI" | ||
| ), | ||
| f"{REMOTE_BASE_IRI}#nt-syntax-bad-uri-04": pytest.mark.xfail( | ||
| reason="accepts an invalid IRI" | ||
| ), | ||
| f"{REMOTE_BASE_IRI}#nt-syntax-bad-uri-05": pytest.mark.xfail( | ||
| reason="accepts an invalid IRI" | ||
| ), | ||
| f"{REMOTE_BASE_IRI}#nt-syntax-bad-esc-01": pytest.mark.xfail( | ||
| reason="accepts badly escaped literal" | ||
| ), | ||
| f"{REMOTE_BASE_IRI}#nt-syntax-bad-esc-02": pytest.mark.xfail( | ||
| reason="accepts badly escaped literal" | ||
| ), | ||
| f"{REMOTE_BASE_IRI}#nt-syntax-bad-esc-03": pytest.mark.xfail( | ||
| reason="accepts badly escaped literal" | ||
| ), | ||
| f"{REMOTE_BASE_IRI}#nt-syntax-bad-esc-04": pytest.mark.xfail( | ||
| reason="accepts badly escaped literal" | ||
| ), | ||
| f"{REMOTE_BASE_IRI}#minimal_whitespace": pytest.mark.xfail( | ||
| reason="Not parsing valid N-Triples syntax." | ||
| ), | ||
| } | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| "rdf_test_uri, type, rdf_test", | ||
| read_manifest( | ||
| os.path.join(TEST_DATA_DIR, "suites", "w3c/ntriples/manifest.ttl"), legacy=True | ||
| ["manifest_entry"], | ||
| params_from_sources( | ||
| MAPPER, | ||
| ManifestEntry, | ||
| LOCAL_BASE_DIR / "manifest.ttl", | ||
| mark_dict=MARK_DICT, | ||
| report_prefix="rdflib_w3c_ntriples", | ||
| ), | ||
| ) | ||
| def test_manifest(rdf_test_uri: URIRef, type: Node, rdf_test: RDFTest): | ||
| testers[type](rdf_test) | ||
| def test_entry(manifest_entry: ManifestEntry) -> None: | ||
| check_entry(manifest_entry) |
Oops, something went wrong.