diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 88d8deaa..53598e4c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,4 +79,4 @@ jobs: ckan -c test.ini db init ckan -c test.ini db pending-migrations --apply - name: Run tests - run: pytest --ckan-ini=test.ini --cov=ckanext.dcat --cov-report=term-missing --cov-append --disable-warnings ckanext/dcat/tests + run: pytest --ckan-ini=test.ini --cov=ckanext.dcat --cov-report=term-missing --cov-append --disable-warnings ckanext/dcat/tests \ No newline at end of file diff --git a/ckanext/dcat/harvesters/base.py b/ckanext/dcat/harvesters/base.py index 64412456..4a5c39bc 100644 --- a/ckanext/dcat/harvesters/base.py +++ b/ckanext/dcat/harvesters/base.py @@ -159,17 +159,12 @@ def _read_datasets_from_db(self, guid): ''' Returns a database result of datasets matching the given guid. ''' - if toolkit.check_ckan_version(max_version="2.11"): - datasets = model.Session.query(model.Package.id) \ - .join(model.PackageExtra) \ - .filter(model.PackageExtra.key == 'guid') \ - .filter(model.PackageExtra.value == guid) \ - .filter(model.Package.state == 'active') \ - .all() - else: - datasets = model.Session.query(model.Package.id) \ - .filter(model.Package.extras['guid'] == f'"{guid}"') \ - .all() + datasets = model.Session.query(model.Package.id) \ + .join(model.PackageExtra) \ + .filter(model.PackageExtra.key == 'guid') \ + .filter(model.PackageExtra.value == guid) \ + .filter(model.Package.state == 'active') \ + .all() return datasets diff --git a/ckanext/dcat/profiles/base.py b/ckanext/dcat/profiles/base.py index efb30d29..30b989e7 100644 --- a/ckanext/dcat/profiles/base.py +++ b/ckanext/dcat/profiles/base.py @@ -536,7 +536,9 @@ def _agents_details(self, subject, predicate): agent_details = {} agent_details["uri"] = str(agent) if isinstance(agent, term.URIRef) else "" agent_details["name"] = self._object_value(agent, FOAF.name) - agent_details["email"] = self._object_value(agent, FOAF.mbox) + agent_details["email"] = self._without_mailto( + self._object_value(agent, FOAF.mbox) + ) if not agent_details["email"]: agent_details["email"] = self._without_mailto( self._object_value(agent, VCARD.hasEmail) @@ -573,6 +575,9 @@ def _contact_details(self, subject, predicate): ) contact["identifier"] = self._get_vcard_property_value(agent, VCARD.hasUID) + + contact["url"] = self._get_vcard_property_value(agent, VCARD.hasURL) + contacts.append(contact) return contacts @@ -818,9 +823,7 @@ def _add_spatial_value_to_graph(self, spatial_ref, predicate, value): or object. """ spatial_formats = aslist( - config.get( - "ckanext.dcat.output_spatial_format", DEFAULT_SPATIAL_FORMATS - ) + config.get("ckanext.dcat.output_spatial_format", DEFAULT_SPATIAL_FORMATS) ) if isinstance(value, str): diff --git a/ckanext/dcat/profiles/euro_dcat_ap_base.py b/ckanext/dcat/profiles/euro_dcat_ap_base.py index 2356a2d4..e42c6afb 100644 --- a/ckanext/dcat/profiles/euro_dcat_ap_base.py +++ b/ckanext/dcat/profiles/euro_dcat_ap_base.py @@ -144,12 +144,12 @@ def _parse_dataset_base(self, dataset_dict, dataset_ref): contact = self._contact_details(dataset_ref, ADMS.contactPoint) if contact: contact = contact[0] - for key in ("uri", "name", "email", "identifier"): + for key in ("uri", "name", "email", "identifier", "url"): if contact.get(key): dataset_dict["extras"].append( { "key": "contact_{0}".format(key), - "value": contact.get(key) + "value": contact.get(key), } ) diff --git a/ckanext/dcat/profiles/euro_dcat_ap_scheming.py b/ckanext/dcat/profiles/euro_dcat_ap_scheming.py index ca72cd21..b8402a65 100644 --- a/ckanext/dcat/profiles/euro_dcat_ap_scheming.py +++ b/ckanext/dcat/profiles/euro_dcat_ap_scheming.py @@ -168,6 +168,13 @@ def _graph_from_dataset_v2_scheming(self, dataset_dict, dataset_ref): "identifier", _type=URIRefOrLiteral, ) + self._add_triple_from_dict( + item, + contact_details, + VCARD.hasURL, + "url", + _type=URIRef, + ) self._add_agents(dataset_ref, dataset_dict, "publisher", DCT.publisher) self._add_agents(dataset_ref, dataset_dict, "creator", DCT.creator) diff --git a/ckanext/dcat/schemas/dcat_ap_full.yaml b/ckanext/dcat/schemas/dcat_ap_full.yaml index 47bc85fe..aee4fffd 100644 --- a/ckanext/dcat/schemas/dcat_ap_full.yaml +++ b/ckanext/dcat/schemas/dcat_ap_full.yaml @@ -46,7 +46,10 @@ dataset_fields: - field_name: identifier label: Identifier help_text: Unique identifier for the contact point. Such as a ROR ID. - + + - field_name: url + label: URL + help_text: A URL associated with the contact help_text: Contact information for enquiries about the dataset. - field_name: publisher diff --git a/ckanext/dcat/schemas/dcat_ap_multilingual.yaml b/ckanext/dcat/schemas/dcat_ap_multilingual.yaml index 63c07c7a..98e9ffb4 100644 --- a/ckanext/dcat/schemas/dcat_ap_multilingual.yaml +++ b/ckanext/dcat/schemas/dcat_ap_multilingual.yaml @@ -104,7 +104,10 @@ dataset_fields: - field_name: identifier label: Identifier help_text: Unique identifier for the contact point. Such as a ROR ID. - + + - field_name: url + label: URL + help_text: A URL associated with the contact help_text: Contact information for enquiries about the dataset. - field_name: license_id diff --git a/ckanext/dcat/schemas/dcat_ap_recommended.yaml b/ckanext/dcat/schemas/dcat_ap_recommended.yaml index ca38ab16..3ab995dd 100644 --- a/ckanext/dcat/schemas/dcat_ap_recommended.yaml +++ b/ckanext/dcat/schemas/dcat_ap_recommended.yaml @@ -42,6 +42,10 @@ dataset_fields: - field_name: email label: Email display_snippet: email.html + + - field_name: url + label: URL + help_text: A URL associated with the contact help_text: Contact information for enquiries about the dataset. - field_name: publisher diff --git a/ckanext/dcat/schemas/dcat_us_full.yaml b/ckanext/dcat/schemas/dcat_us_full.yaml index 24e8dedd..6f55903f 100644 --- a/ckanext/dcat/schemas/dcat_us_full.yaml +++ b/ckanext/dcat/schemas/dcat_us_full.yaml @@ -42,6 +42,10 @@ dataset_fields: - field_name: email label: Email display_snippet: email.html + + - field_name: url + label: URL + help_text: A URL associated with the contact help_text: Contact information for enquiries about the dataset. - field_name: publisher diff --git a/ckanext/dcat/schemas/dcat_us_recommended.yaml b/ckanext/dcat/schemas/dcat_us_recommended.yaml index f5aea9b3..0cacdb53 100644 --- a/ckanext/dcat/schemas/dcat_us_recommended.yaml +++ b/ckanext/dcat/schemas/dcat_us_recommended.yaml @@ -42,6 +42,10 @@ dataset_fields: - field_name: email label: Email display_snippet: email.html + + - field_name: url + label: URL + help_text: A URL associated with the contact help_text: Contact information for enquiries about the dataset. - field_name: publisher diff --git a/ckanext/dcat/tests/profiles/dcat_ap_3/test_euro_dcatap_3_profile_parse.py b/ckanext/dcat/tests/profiles/dcat_ap_3/test_euro_dcatap_3_profile_parse.py index b54774a2..25cd2c64 100644 --- a/ckanext/dcat/tests/profiles/dcat_ap_3/test_euro_dcatap_3_profile_parse.py +++ b/ckanext/dcat/tests/profiles/dcat_ap_3/test_euro_dcatap_3_profile_parse.py @@ -91,6 +91,7 @@ def test_e2e_dcat_to_ckan(self): assert dataset["contact"][0]["name"] == "Point of Contact" assert dataset["contact"][0]["email"] == "contact@some.org" + assert dataset["contact"][0]["url"] == "https://example.org" assert ( dataset["publisher"][0]["name"] == "Publishing Organization for dataset 1" diff --git a/ckanext/dcat/tests/profiles/dcat_ap_3/test_euro_dcatap_3_profile_serialize.py b/ckanext/dcat/tests/profiles/dcat_ap_3/test_euro_dcatap_3_profile_serialize.py index 75d359a3..b509e37d 100644 --- a/ckanext/dcat/tests/profiles/dcat_ap_3/test_euro_dcatap_3_profile_serialize.py +++ b/ckanext/dcat/tests/profiles/dcat_ap_3/test_euro_dcatap_3_profile_serialize.py @@ -148,6 +148,9 @@ def test_e2e_ckan_to_dcat(self): VCARD.hasUID, dataset_dict["contact"][0]["identifier"], ) + assert self._triple( + g, contact_details[0][2], VCARD.hasURL, URIRef(dataset_dict["contact"][0]["url"]) + ) assert self._triple( g, contact_details[1][2], VCARD.fn, dataset_dict["contact"][1]["name"] ) diff --git a/ckanext/dcat/tests/shacl/test_shacl.py b/ckanext/dcat/tests/shacl/test_shacl.py index 16e2ff3b..f3780905 100644 --- a/ckanext/dcat/tests/shacl/test_shacl.py +++ b/ckanext/dcat/tests/shacl/test_shacl.py @@ -199,9 +199,6 @@ def test_validate_dcat_us_3_graph(): graph = graph_from_dataset("ckan_full_dataset_dcat_us_vocabularies.json") - graph.serialize(destination="graph.ttl") - - graph.serialize(destination="graph.xml") path = _get_shacl_file_path("dcat-us_3.0_shacl_shapes.ttl") r = validate(graph, shacl_graph=path) conforms, results_graph, results_text = r diff --git a/examples/ckan/ckan_full_dataset_dcat_ap.json b/examples/ckan/ckan_full_dataset_dcat_ap.json index 922e947d..6adb770d 100644 --- a/examples/ckan/ckan_full_dataset_dcat_ap.json +++ b/examples/ckan/ckan_full_dataset_dcat_ap.json @@ -53,7 +53,8 @@ { "name": "Contact 1", "email": "contact1@example.org", - "identifier": "123" + "identifier": "123", + "url": "https://example.org" }, { "name": "Contact 2", diff --git a/examples/dcat/dataset.rdf b/examples/dcat/dataset.rdf index 7ab556cb..ff280d9a 100644 --- a/examples/dcat/dataset.rdf +++ b/examples/dcat/dataset.rdf @@ -68,6 +68,7 @@ Point of Contact +