Merge branch 'main' into yatte2-conflicts

.github/workflows/testgha.yml

@@ -20,12 +20,11 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools wheel pytest DeepDiff
-        pip install -r requirements.txt
+        pip install poetry
+        poetry update
     - name: Build yatter
       run: |
-        cd src
-        pip install .
+        poetry install --with test
     - name: Test with pytest
       run: |
-        pytest
+        poetry run pytest

.gitignore

@@ -7,4 +7,6 @@ src/build
 src/dist
 src/yatter.egg-info
 api
-.coverage
+.coverage
+src/yatter/__pycache__
+*.pyc

pyproject.toml

@@ -0,0 +1,24 @@
+[tool.poetry]
+name = "yatter"
+version = "1.1.4"
+description = "The tool translates mapping rules from YARRRML in a turtle-based serialization of RML or R2RML."
+authors = ["David Chaves <[email protected]>", "Rubén Buján <[email protected]>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.8"
+ruamel-yaml = "^0.17.21"
+rdflib = "^6.2.0"
+coloredlogs = "^15.0.1"
+
+[tool.poetry.group.test.dependencies]
+pytest = "^8.3.3"
+setuptools = "^75.1.0"
+wheel = "^0.44.0"
+DeepDiff = "^8.0.1"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+

src/yatter/__init__.py

@@ -1,30 +1,32 @@
 from .constants import *
 from .mapping import add_prefix, add_mapping, add_inverse_prefix, get_non_asserted_mappings, merge_mapping_section_by_key
-from .source import get_initial_sources, add_source, generate_database_connections, add_table, add_inverse_source
+from .source import add_source, generate_database_connections, add_table, add_inverse_source
 from .subject import add_subject, add_inverse_subject
 from .predicateobject import add_predicate_object_maps, add_inverse_pom
 from .target import add_logical_targets
 from .function import add_functions
+from .normalization import normalize
 import rdflib
 import ruamel.yaml as yaml
 
 
 def translate(yarrrml_data, mapping_format=RML_URI):
     logger.info("Translating YARRRML mapping to [R2]RML")
-
-    list_initial_sources = get_initial_sources(yarrrml_data)
+    external_sources = {}
+    external_targets = {}
+    yarrrml_data = normalize(yarrrml_data, external_sources, external_targets)
     rml_mapping = [add_prefix(yarrrml_data)]
-    rml_mapping.extend(generate_database_connections(yarrrml_data, list_initial_sources))
-    rml_mapping.extend(add_logical_targets(yarrrml_data))
+    rml_mapping.extend(generate_database_connections(yarrrml_data, external_sources))
+    rml_mapping.extend(add_logical_targets(yarrrml_data, external_targets))
     rml_mapping.extend(add_functions(yarrrml_data))
-    external_refs = []
+
     try:
         mappings, mapping_format = get_non_asserted_mappings(yarrrml_data, mapping_format)
         for mapping in yarrrml_data.get(YARRRML_MAPPINGS):
             if mapping_format == R2RML_URI:
-                source_list = add_table(yarrrml_data, mapping, list_initial_sources)
+                source_list = add_table(yarrrml_data, mapping)
             else:
-                source_list, external_refs = add_source(yarrrml_data, mapping, list_initial_sources)
+                source_list = add_source(yarrrml_data, mapping, external_sources)
             subject_list = add_subject(yarrrml_data, mapping, mapping_format)
             pred = add_predicate_object_maps(yarrrml_data, mapping, mapping_format)
             it = 0
@@ -77,7 +79,7 @@ def inverse_translation(rdf_mapping, mapping_format=RML_URI):
         yarrrml_tm = {YARRRML_SOURCES: [add_inverse_source(tm, rdf_mapping, mapping_format)]}
         subject, classes = add_inverse_subject(tm, rdf_mapping)
         yarrrml_tm.update(subject)
-        yarrrml_tm[YARRRML_PREDICATEOBJECT_SHORTCUT] = add_inverse_pom(tm, rdf_mapping, classes, yarrrml_mapping[YARRRML_PREFIXES])
+        yarrrml_tm['po'] = add_inverse_pom(tm, rdf_mapping, classes, yarrrml_mapping[YARRRML_PREFIXES])
         yarrrml_mapping[YARRRML_MAPPINGS][tm_name] = yarrrml_tm
 
     logger.info("Translation has finished successfully.")

src/yatter/constants.py

@@ -54,6 +54,11 @@
 RML_VALUE_MAP = 'fnml:valueMap'
 RML_VALUE_MAP_CLASS = 'fnml:ValueMap'
 
+RML_CC_GATHER = 'rml:gather'
+RML_CC_GATHER_AS = 'rml:gatherAs'
+RML_CC_EMPTY = 'rml:allowEmptyListAndContainer'
+RML_CC_STRATEGY = 'rml:strategy'
+
 STAR_CLASS = 'rml:StarMap'
 STAR_NON_ASSERTED_CLASS = 'rml:NonAssertedTriplesMap'
 STAR_QUOTED = 'rml:quotedTriplesMap'
@@ -121,7 +126,6 @@
 
 YARRRML_PREFIXES = 'prefixes'
 YARRRML_SOURCES = 'sources'
-YARRRML_SOURCE = 'source'
 YARRRML_TABLE = 'table'
 YARRRML_ACCESS = 'access'
 YARRRML_QUERY = 'query'
@@ -140,35 +144,24 @@
 
 YARRRML_MAPPINGS = 'mappings' # used for mappings in conditions and mappings main key
 YARRRML_MAPPING = 'mapping'
-YARRRRL_MAPPINGS_SHORTCUT = 'm'
 
-YARRRML_SUBJECTS_SHORTCUT = 's'
-YARRRML_SUBJECT = 'subject'
 YARRRML_SUBJECTS = 'subjects'
-
+YARRRML_AUTHORS = 'authors'
 YARRRML_GRAPHS = 'graphs'
-YARRRML_GRAPH = 'graph'
-YARRRML_GRAPH_SHORTCUT = 'g'
 
 YARRRML_PREDICATEOBJECT = 'predicateobjects'
-YARRRML_PREDICATEOBJECT_SHORTCUT = 'po'
 
 YARRRML_PREDICATES = 'predicates'
-YARRRML_PREDICATE = 'predicate'
-YARRRML_PREDICATES_SHORTCUT = 'p'
 YARRRML_OBJECTS = 'objects'
-YARRRML_OBJECT = 'objects'
-YARRRML_OBJECT_SHORTCUT = 'o'
 YARRRML_VALUE = 'value'
-YARRRML_VALUE_SHORTCUT = 'v'
 YARRRML_DATATYPE = 'datatype'
 YARRRML_LANGUAGE = 'language'
 
 YARRRML_CONDITION = 'condition'
 YARRRML_EQUAL = 'equal'
 YARRRML_JOIN = 'join'
 YARRRML_PARAMETERS = 'parameters' #used for conditions and functions
-YARRRML_PARAMETERS_SHORTCUT = 'pms'
+
 
 YARRRML_IRI = '~iri'
 YARRRML_LANG = '~lang'
@@ -182,22 +175,25 @@
 YARRRML_COMPRESSION = 'compression'
 
 YARRRML_FUNCTION = 'function'
-YARRRML_FUNCTION_SHORTCUT_A = 'fn'
-YARRRML_FUNCTION_SHORTCUT_B = 'f'
 
 YARRRML_PARAMETER = 'parameter'
-YARRRML_PARAMETER_SHORTCUT = 'p'
 
-YARRRML_MAPPING_KEYS = [YARRRML_MAPPINGS, YARRRML_MAPPING, YARRRRL_MAPPINGS_SHORTCUT]
-YARRRML_SUBJECT_KEYS = [YARRRML_SUBJECTS, YARRRML_SUBJECT, YARRRML_SUBJECTS_SHORTCUT]
-YARRRML_POM_KEYS = [YARRRML_PREDICATEOBJECT, YARRRML_PREDICATEOBJECT_SHORTCUT]
-YARRRML_GRAPH_KEYS = [YARRRML_GRAPHS, YARRRML_GRAPH, YARRRML_GRAPH_SHORTCUT]
-YARRRML_PREDICATE_KEYS = [YARRRML_PREDICATES, YARRRML_PREDICATE, YARRRML_PREDICATES_SHORTCUT]
-YARRRML_OBJECT_KEYS = [YARRRML_OBJECTS, YARRRML_OBJECT, YARRRML_OBJECT_SHORTCUT]
-YARRRML_FUNCTION_KEYS = [YARRRML_FUNCTION, YARRRML_FUNCTION_SHORTCUT_A, YARRRML_FUNCTION_SHORTCUT_B]
-YARRRML_PARAMETERS_KEYS = [YARRRML_PARAMETERS, YARRRML_PARAMETERS_SHORTCUT]
-YARRRML_PARAMETER_KEYS = [YARRRML_PARAMETER, YARRRML_PARAMETER_SHORTCUT]
-YARRRML_VALUE_KEYS = [YARRRML_VALUE, YARRRML_VALUE_SHORTCUT]
+YARRRML_GATHER = 'gather'
+YARRRML_GATHER_AS = 'gatherAs'
+YARRRML_GATHER_AS_OPTIONS = ["alt","bag","seq","list"]
+YARRRML_EMPTY = 'empty'
+YARRRML_STRATEGY = 'strategy'
+
+YARRRML_MAPPING_KEYS = [YARRRML_MAPPINGS, YARRRML_MAPPING]
+YARRRML_SUBJECT_KEYS = [YARRRML_SUBJECTS]
+YARRRML_POM_KEYS = [YARRRML_PREDICATEOBJECT]
+YARRRML_GRAPH_KEYS = [YARRRML_GRAPHS]
+YARRRML_PREDICATE_KEYS = [YARRRML_PREDICATES]
+YARRRML_OBJECT_KEYS = [YARRRML_OBJECTS]
+YARRRML_FUNCTION_KEYS = [YARRRML_FUNCTION]
+YARRRML_PARAMETERS_KEYS = [YARRRML_PARAMETERS]
+YARRRML_PARAMETER_KEYS = [YARRRML_PARAMETER]
+YARRRML_VALUE_KEYS = [YARRRML_VALUE]
 
 
 YARRRML_OUTPUT_FORMAT = {
@@ -242,4 +238,4 @@
 }
 
 logger = logging.getLogger(__name__)
-coloredlogs.install(level='DEBUG', fmt='%(asctime)s,%(msecs)03d | %(levelname)s: %(message)s')
+coloredlogs.install(level='DEBUG', fmt='%(asctime)s,%(msecs)03d | %(levelname)s: %(message)s')

src/yatter/function.py

@@ -139,10 +139,15 @@ def generate_extended_join(yarrrml_data):
 
     for value in equals:
         parameters = []
-        conditions = value.replace("(","",1).rsplit(")",1)[0].split(",")
-        parameters.append(['str1',conditions[0]])
-        parameters.append(['str2', conditions[1]])
-        extended_join[YARRRML_CONDITION].append({YARRRML_FUNCTION: YARRRML_EQUAL, YARRRML_PARAMETERS: parameters})
+        for param_name, condition in zip(['str1', 'str2'], value.replace("(", "", 1).rsplit(")", 1)[0].split(",")):
+            parameters.append({
+                YARRRML_PARAMETER: param_name,
+                YARRRML_VALUE: condition
+            })
+        extended_join[YARRRML_CONDITION].append({
+            YARRRML_FUNCTION: YARRRML_EQUAL,
+            YARRRML_PARAMETERS: parameters
+        })
 
     return extended_join
 

src/yatter/mapping.py

@@ -1,6 +1,8 @@
-from .import *
+from . import *
+
 prefixes = {}
 
+
 def add_mapping(mapping, mappings, it):
     map_template = "<" + mapping + "_" + str(it) + "> a "
     if mappings[mapping]:
@@ -58,7 +60,6 @@ def add_prefix(data):
 
 
 def check_common_prefixes(prefix_uri, common_prefixes):
-
     if prefix_uri == R2RML_URI:
         common_prefixes.append("r2rml")
     elif prefix_uri == RML_URI:
@@ -106,8 +107,8 @@ def get_non_asserted_mappings(yarrrml_data, mapping_format):
                 for value in yarrrml_data.get(YARRRML_MAPPINGS).get(mapping).get(key):
                     if type(value) is dict:
                         star_data = [value]
-                        if YARRRML_OBJECT_SHORTCUT in value:
-                            star_data = value[YARRRML_OBJECT_SHORTCUT]
+                        if YARRRML_OBJECTS in value:
+                            star_data = value[YARRRML_OBJECTS]
                         for val in star_data:
                             if YARRRML_NON_ASSERTED in val:
                                 mappings[val[YARRRML_NON_ASSERTED]] = "non_asserted"
@@ -118,9 +119,8 @@ def get_non_asserted_mappings(yarrrml_data, mapping_format):
     return mappings, mapping_format
 
 
-
-def merge_mapping_section_by_key(key,yarrrml_list):
-    output = {key:{}}
+def merge_mapping_section_by_key(key, yarrrml_list):
+    output = {key: {}}
     for yarrrml_mapping in yarrrml_list:
         output[key] = output[key] | yarrrml_mapping
-    return output
+    return output