From 2e718948107ef8622095b84c2aa4413985e48801 Mon Sep 17 00:00:00 2001 From: Mark van der Pas Date: Sun, 8 May 2022 11:15:54 +0200 Subject: [PATCH 01/12] Refactor service json results parser to fix issue 1278 and align with sparql/results/jsonresults parser. --- rdflib/plugins/sparql/evaluate.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/rdflib/plugins/sparql/evaluate.py b/rdflib/plugins/sparql/evaluate.py index 6cdbe7540..278660fd2 100644 --- a/rdflib/plugins/sparql/evaluate.py +++ b/rdflib/plugins/sparql/evaluate.py @@ -381,18 +381,18 @@ def _yieldBindingsFromServiceCallResult(ctx: QueryContext, r, variables): res_dict: Dict[Variable, Identifier] = {} for var in variables: if var in r and r[var]: - if r[var]["type"] == "uri": - res_dict[Variable(var)] = URIRef(r[var]["value"]) - elif r[var]["type"] == "bnode": - res_dict[Variable(var)] = BNode(r[var]["value"]) - elif r[var]["type"] == "literal" and "datatype" in r[var]: - res_dict[Variable(var)] = Literal( - r[var]["value"], datatype=r[var]["datatype"] - ) - elif r[var]["type"] == "literal" and "xml:lang" in r[var]: - res_dict[Variable(var)] = Literal( - r[var]["value"], lang=r[var]["xml:lang"] - ) + d = r[var] + t = d["type"] + if t == "uri": + res_dict[Variable(var)] = URIRef(d["value"]) + elif t == "literal": + res_dict[Variable(var)] = Literal(d["value"], datatype=d.get("datatype"), lang=d.get("xml:lang")) + elif t == "typed-literal": + res_dict[Variable(var)] = Literal(d["value"], datatype=URIRef(d["datatype"])) + elif t == "bnode": + res_dict[Variable(var)] = BNode(d["value"]) + else: + raise NotImplementedError("json term type %r" % t) yield FrozenBindings(ctx, res_dict) From f6d3e13697d4b3ad832d973b99d08dfa7bbd0f4d Mon Sep 17 00:00:00 2001 From: Mark van der Pas Date: Sun, 8 May 2022 11:16:36 +0200 Subject: [PATCH 02/12] Add test for issue 1278 literals returned as null --- test/test_issues/test_issue1278.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 test/test_issues/test_issue1278.py diff --git a/test/test_issues/test_issue1278.py b/test/test_issues/test_issue1278.py new file mode 100644 index 000000000..9d38d5106 --- /dev/null +++ b/test/test_issues/test_issue1278.py @@ -0,0 +1,18 @@ +from test.utils import helper +from rdflib import Graph, Literal, Variable + + +def test(): + """Test service returns simple literals not as NULL. + + Issue: https://github.com/RDFLib/rdflib/issues/1278 + """ + + g = Graph() + q = """SELECT ?s ?p ?o +WHERE { + SERVICE { + VALUES (?s ?p ?o) {( "c")} + } +}""" + assert results.bindings[0].get(Variable('o')) == Literal('c') From e0105081578dc13aaa79032a533dcdc74937328f Mon Sep 17 00:00:00 2001 From: Mark van der Pas Date: Sun, 8 May 2022 11:32:06 +0200 Subject: [PATCH 03/12] Fix formatting issue --- test/test_issues/test_issue1278.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_issues/test_issue1278.py b/test/test_issues/test_issue1278.py index 9d38d5106..f4a26f2f1 100644 --- a/test/test_issues/test_issue1278.py +++ b/test/test_issues/test_issue1278.py @@ -15,4 +15,4 @@ def test(): VALUES (?s ?p ?o) {( "c")} } }""" - assert results.bindings[0].get(Variable('o')) == Literal('c') + assert results.bindings[0].get(Variable("o")) == Literal("c") From 17bf257958965b2139b0a69895be7122aaaea860 Mon Sep 17 00:00:00 2001 From: Mark van der Pas Date: Sun, 8 May 2022 11:45:03 +0200 Subject: [PATCH 04/12] Fix formatting issue evaluate.py --- rdflib/plugins/sparql/evaluate.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rdflib/plugins/sparql/evaluate.py b/rdflib/plugins/sparql/evaluate.py index 278660fd2..d63f07fea 100644 --- a/rdflib/plugins/sparql/evaluate.py +++ b/rdflib/plugins/sparql/evaluate.py @@ -386,9 +386,13 @@ def _yieldBindingsFromServiceCallResult(ctx: QueryContext, r, variables): if t == "uri": res_dict[Variable(var)] = URIRef(d["value"]) elif t == "literal": - res_dict[Variable(var)] = Literal(d["value"], datatype=d.get("datatype"), lang=d.get("xml:lang")) + res_dict[Variable(var)] = Literal( + d["value"], datatype=d.get("datatype"), lang=d.get("xml:lang") + ) elif t == "typed-literal": - res_dict[Variable(var)] = Literal(d["value"], datatype=URIRef(d["datatype"])) + res_dict[Variable(var)] = Literal( + d["value"], datatype=URIRef(d["datatype"]) + ) elif t == "bnode": res_dict[Variable(var)] = BNode(d["value"]) else: From 5ef9bdd0bf9e8ade0dcb30e081cf172ac13e9bfd Mon Sep 17 00:00:00 2001 From: Mark van der Pas Date: Sun, 8 May 2022 11:57:22 +0200 Subject: [PATCH 05/12] Fix imports formatting issue --- test/test_issues/test_issue1278.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_issues/test_issue1278.py b/test/test_issues/test_issue1278.py index f4a26f2f1..57315fea1 100644 --- a/test/test_issues/test_issue1278.py +++ b/test/test_issues/test_issue1278.py @@ -1,4 +1,5 @@ from test.utils import helper + from rdflib import Graph, Literal, Variable From ccbdbc2d2342d42c09405b934d8a9fe1e2cf3933 Mon Sep 17 00:00:00 2001 From: Mark van der Pas Date: Sun, 8 May 2022 17:57:52 +0200 Subject: [PATCH 06/12] fix test for literals returned as Null (#1278) --- test/test_issues/test_issue1278.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_issues/test_issue1278.py b/test/test_issues/test_issue1278.py index 57315fea1..35b000cc2 100644 --- a/test/test_issues/test_issue1278.py +++ b/test/test_issues/test_issue1278.py @@ -16,4 +16,5 @@ def test(): VALUES (?s ?p ?o) {( "c")} } }""" + results = helper.query_with_retry(g, q) assert results.bindings[0].get(Variable("o")) == Literal("c") From dde8b2d314959f89837a0cdb656ad6dda5aa0d1d Mon Sep 17 00:00:00 2001 From: Mark van der Pas Date: Wed, 11 May 2022 16:59:22 +0200 Subject: [PATCH 07/12] Add test for SERVICE returning different node types --- test/test_sparql/test_service.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/test_sparql/test_service.py b/test/test_sparql/test_service.py index 9798f3df8..89e279f3f 100644 --- a/test/test_sparql/test_service.py +++ b/test/test_sparql/test_service.py @@ -135,6 +135,32 @@ def test_service_with_implicit_select_and_allcaps(): assert len(results) == 3 +def test_service_node_types(): + """Test if SERVICE properly returns different types of nodes: + - URI; + - Simple Literal; + - Literal with datatype ; + - Literal with language tag . + """ + + g = Graph() + q = """ +SELECT ?o +WHERE { + SERVICE { + VALUES (?s ?p ?o) { + ( ) + ( "Simple Literal") + ( "String Literal"^^xsd:string) + ( "String Language"@en) + } + } + FILTER( ?o IN (, "Simple Literal", "String Literal", "String Language"@en) ) +}""" + results = helper.query_with_retry(g, q) + assert len(results.bindings) == 4 + + # def test_with_fixture(httpserver): # httpserver.expect_request("/sparql/?query=SELECT * WHERE ?s ?p ?o").respond_with_json({"vars": ["s","p","o"], "bindings":[]}) # test_server = httpserver.url_for('/sparql') @@ -151,3 +177,4 @@ def test_service_with_implicit_select_and_allcaps(): test_service_with_implicit_select() test_service_with_implicit_select_and_prefix() test_service_with_implicit_select_and_base() + test_service_node_types() From 01f4515cb0ff0e3946fbe895dd583b93b058afe7 Mon Sep 17 00:00:00 2001 From: Mark van der Pas Date: Wed, 11 May 2022 17:03:51 +0200 Subject: [PATCH 08/12] fix comment formatting --- test/test_sparql/test_service.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_sparql/test_service.py b/test/test_sparql/test_service.py index 89e279f3f..1a0f50bec 100644 --- a/test/test_sparql/test_service.py +++ b/test/test_sparql/test_service.py @@ -137,10 +137,10 @@ def test_service_with_implicit_select_and_allcaps(): def test_service_node_types(): """Test if SERVICE properly returns different types of nodes: - - URI; - - Simple Literal; - - Literal with datatype ; - - Literal with language tag . + - URI; + - Simple Literal; + - Literal with datatype ; + - Literal with language tag . """ g = Graph() From ad223704065fd398f57ce316091ccce6f91c3118 Mon Sep 17 00:00:00 2001 From: Mark van der Pas Date: Thu, 12 May 2022 10:18:01 +0200 Subject: [PATCH 09/12] fix test data type --- test/test_sparql/test_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_sparql/test_service.py b/test/test_sparql/test_service.py index 1a0f50bec..53c345236 100644 --- a/test/test_sparql/test_service.py +++ b/test/test_sparql/test_service.py @@ -155,10 +155,10 @@ def test_service_node_types(): ( "String Language"@en) } } - FILTER( ?o IN (, "Simple Literal", "String Literal", "String Language"@en) ) + FILTER( ?o IN (, "Simple Literal", "String Literal"^^xsd:string, "String Language"@en) ) }""" results = helper.query_with_retry(g, q) - assert len(results.bindings) == 4 + assert len(results) == 4 # def test_with_fixture(httpserver): From c395200a01fddba62633483d560d9777856071f7 Mon Sep 17 00:00:00 2001 From: Iwan Aucamp Date: Thu, 12 May 2022 20:21:07 +0200 Subject: [PATCH 10/12] move test from test_issue1278 to test_service --- test/test_issues/test_issue1278.py | 20 -------------------- test/test_sparql/test_service.py | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 20 deletions(-) delete mode 100644 test/test_issues/test_issue1278.py diff --git a/test/test_issues/test_issue1278.py b/test/test_issues/test_issue1278.py deleted file mode 100644 index 35b000cc2..000000000 --- a/test/test_issues/test_issue1278.py +++ /dev/null @@ -1,20 +0,0 @@ -from test.utils import helper - -from rdflib import Graph, Literal, Variable - - -def test(): - """Test service returns simple literals not as NULL. - - Issue: https://github.com/RDFLib/rdflib/issues/1278 - """ - - g = Graph() - q = """SELECT ?s ?p ?o -WHERE { - SERVICE { - VALUES (?s ?p ?o) {( "c")} - } -}""" - results = helper.query_with_retry(g, q) - assert results.bindings[0].get(Variable("o")) == Literal("c") diff --git a/test/test_sparql/test_service.py b/test/test_sparql/test_service.py index 53c345236..ded3db299 100644 --- a/test/test_sparql/test_service.py +++ b/test/test_sparql/test_service.py @@ -135,6 +135,23 @@ def test_service_with_implicit_select_and_allcaps(): assert len(results) == 3 +def test_simple_not_null(): + """Test service returns simple literals not as NULL. + + Issue: https://github.com/RDFLib/rdflib/issues/1278 + """ + + g = Graph() + q = """SELECT ?s ?p ?o +WHERE { + SERVICE { + VALUES (?s ?p ?o) {( "c")} + } +}""" + results = helper.query_with_retry(g, q) + assert results.bindings[0].get(Variable("o")) == Literal("c") + + def test_service_node_types(): """Test if SERVICE properly returns different types of nodes: - URI; From 8c1a293c5af37240cba425510569968cb796b255 Mon Sep 17 00:00:00 2001 From: Iwan Aucamp Date: Thu, 12 May 2022 22:34:47 +0200 Subject: [PATCH 11/12] Add more tests Also make existing tests stricter. --- test/test_sparql/test_service.py | 156 +++++++++++++++++++++++++++++-- 1 file changed, 147 insertions(+), 9 deletions(-) diff --git a/test/test_sparql/test_service.py b/test/test_sparql/test_service.py index ded3db299..31ba6e8a5 100644 --- a/test/test_sparql/test_service.py +++ b/test/test_sparql/test_service.py @@ -1,8 +1,31 @@ +import json +from contextlib import ExitStack from test.utils import helper +from test.utils.httpservermock import ( + MethodName, + MockHTTPResponse, + ServedBaseHTTPServerMock, +) +from typing import ( + Dict, + FrozenSet, + Generator, + List, + Mapping, + Optional, + Sequence, + Tuple, + Type, + Union, +) + +import pytest from rdflib import Graph, Literal, URIRef, Variable from rdflib.compare import isomorphic +from rdflib.namespace import XSD from rdflib.plugins.sparql import prepareQuery +from rdflib.term import BNode, Identifier def test_service(): @@ -135,6 +158,15 @@ def test_service_with_implicit_select_and_allcaps(): assert len(results) == 3 +def freeze_bindings( + bindings: Sequence[Mapping[Variable, Identifier]] +) -> FrozenSet[FrozenSet[Tuple[Variable, Identifier]]]: + result = [] + for binding in bindings: + result.append(frozenset(((key, value)) for key, value in binding.items())) + return frozenset(result) + + def test_simple_not_null(): """Test service returns simple literals not as NULL. @@ -170,21 +202,127 @@ def test_service_node_types(): ( "Simple Literal") ( "String Literal"^^xsd:string) ( "String Language"@en) + ( "String Language"@en) } } FILTER( ?o IN (, "Simple Literal", "String Literal"^^xsd:string, "String Language"@en) ) }""" results = helper.query_with_retry(g, q) - assert len(results) == 4 - -# def test_with_fixture(httpserver): -# httpserver.expect_request("/sparql/?query=SELECT * WHERE ?s ?p ?o").respond_with_json({"vars": ["s","p","o"], "bindings":[]}) -# test_server = httpserver.url_for('/sparql') -# g = Graph() -# q = 'SELECT * WHERE {SERVICE <'+test_server+'>{?s ?p ?o} . ?s ?p ?o .}' -# results = g.query(q) -# assert len(results) == 0 + expected = freeze_bindings( + [ + {Variable('o'): URIRef('http://example.org/URI')}, + {Variable('o'): Literal('Simple Literal')}, + { + Variable('o'): Literal( + 'String Literal', + datatype=URIRef('http://www.w3.org/2001/XMLSchema#string'), + ) + }, + {Variable('o'): Literal('String Language', lang='en')}, + ] + ) + assert expected == freeze_bindings(results.bindings) + + +@pytest.fixture(scope="module") +def module_httpmock() -> Generator[ServedBaseHTTPServerMock, None, None]: + with ServedBaseHTTPServerMock() as httpmock: + yield httpmock + + +@pytest.fixture(scope="function") +def httpmock( + module_httpmock: ServedBaseHTTPServerMock, +) -> Generator[ServedBaseHTTPServerMock, None, None]: + module_httpmock.reset() + yield module_httpmock + + +@pytest.mark.parametrize( + ("response_bindings", "expected_result"), + [ + ( + [ + {"type": "uri", "value": "http://example.org/uri"}, + {"type": "literal", "value": "literal without type or lang"}, + {"type": "literal", "value": "literal with lang", "xml:lang": "en"}, + { + "type": "typed-literal", + "value": "typed-literal with datatype", + "datatype": f"{XSD.string}", + }, + { + "type": "literal", + "value": "literal with datatype", + "datatype": f"{XSD.string}", + }, + {"type": "bnode", "value": "ohci6Te6aidooNgo"}, + ], + [ + URIRef('http://example.org/uri'), + Literal('literal without type or lang'), + Literal('literal with lang', lang='en'), + Literal( + 'typed-literal with datatype', + datatype=URIRef('http://www.w3.org/2001/XMLSchema#string'), + ), + Literal('literal with datatype', datatype=XSD.string), + BNode('ohci6Te6aidooNgo'), + ], + ), + ( + [ + {"type": "invalid-type"}, + ], + ValueError, + ), + ], +) +def test_with_mock( + httpmock: ServedBaseHTTPServerMock, + response_bindings: List[Dict[str, str]], + expected_result: Union[List[Identifier], Type[Exception]], +) -> None: + """ + This tests that bindings for a variable named var + """ + graph = Graph() + query = """ + PREFIX ex: + SELECT ?var + WHERE { + SERVICE { + ex:s ex:p ?var + } + } + """ + query = query.replace("REMOTE_URL", httpmock.url) + response = { + "head": {"vars": ["var"]}, + "results": {"bindings": [{"var": item} for item in response_bindings]}, + } + httpmock.responses[MethodName.GET].append( + MockHTTPResponse( + 200, + "OK", + json.dumps(response).encode("utf-8"), + {"Content-Type": ["application/sparql-results+json"]}, + ) + ) + catcher: Optional[pytest.ExceptionInfo[Exception]] = None + + with ExitStack() as xstack: + if isinstance(expected_result, type) and issubclass(expected_result, Exception): + catcher = xstack.enter_context(pytest.raises(expected_result)) + else: + expected_bindings = [{Variable("var"): item} for item in expected_result] + bindings = graph.query(query).bindings + if catcher is not None: + assert catcher is not None + assert catcher.value is not None + else: + assert expected_bindings == bindings if __name__ == "__main__": From e71b4a6141e29b625757bb6f991b35c7183957dc Mon Sep 17 00:00:00 2001 From: Iwan Aucamp Date: Thu, 12 May 2022 22:36:11 +0200 Subject: [PATCH 12/12] Some minor improvements - Use longer variable names so it is easier to read - Add type hints - Add comment explaining the reason for `typed-literal` - Improved exception in case of invalid/unsupported type. --- rdflib/plugins/sparql/evaluate.py | 35 ++++++++++++++++++------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/rdflib/plugins/sparql/evaluate.py b/rdflib/plugins/sparql/evaluate.py index d63f07fea..bc5c1448a 100644 --- a/rdflib/plugins/sparql/evaluate.py +++ b/rdflib/plugins/sparql/evaluate.py @@ -18,7 +18,7 @@ import itertools import json as j import re -from typing import Any, Deque, Dict, List, Union +from typing import Any, Deque, Dict, Generator, List, Sequence, Union from urllib.parse import urlencode from urllib.request import Request, urlopen @@ -337,7 +337,8 @@ def evalServiceQuery(ctx: QueryContext, part): res = json["results"]["bindings"] if len(res) > 0: for r in res: - for bound in _yieldBindingsFromServiceCallResult(ctx, r, variables): + # type error: Argument 2 to "_yieldBindingsFromServiceCallResult" has incompatible type "str"; expected "Dict[str, Dict[str, str]]" + for bound in _yieldBindingsFromServiceCallResult(ctx, r, variables): # type: ignore[arg-type] yield bound else: raise Exception( @@ -377,26 +378,32 @@ def _buildQueryStringForServiceCall(ctx: QueryContext, match): return service_query -def _yieldBindingsFromServiceCallResult(ctx: QueryContext, r, variables): +def _yieldBindingsFromServiceCallResult( + ctx: QueryContext, r: Dict[str, Dict[str, str]], variables: List[str] +) -> Generator[FrozenBindings, None, None]: res_dict: Dict[Variable, Identifier] = {} for var in variables: if var in r and r[var]: - d = r[var] - t = d["type"] - if t == "uri": - res_dict[Variable(var)] = URIRef(d["value"]) - elif t == "literal": + var_binding = r[var] + var_type = var_binding["type"] + if var_type == "uri": + res_dict[Variable(var)] = URIRef(var_binding["value"]) + elif var_type == "literal": res_dict[Variable(var)] = Literal( - d["value"], datatype=d.get("datatype"), lang=d.get("xml:lang") + var_binding["value"], + datatype=var_binding.get("datatype"), + lang=var_binding.get("xml:lang"), ) - elif t == "typed-literal": + # This is here because of + # https://www.w3.org/TR/2006/NOTE-rdf-sparql-json-res-20061004/#variable-binding-results + elif var_type == "typed-literal": res_dict[Variable(var)] = Literal( - d["value"], datatype=URIRef(d["datatype"]) + var_binding["value"], datatype=URIRef(var_binding["datatype"]) ) - elif t == "bnode": - res_dict[Variable(var)] = BNode(d["value"]) + elif var_type == "bnode": + res_dict[Variable(var)] = BNode(var_binding["value"]) else: - raise NotImplementedError("json term type %r" % t) + raise ValueError(f"invalid type {var_type!r} for variable {var!r}") yield FrozenBindings(ctx, res_dict)