From 583afbeb4e3f6ae919abe8d596f7209906caefb0 Mon Sep 17 00:00:00 2001 From: Mark Jordan Date: Tue, 5 May 2020 07:05:05 -0700 Subject: [PATCH] WIP on #47. --- modules/field_entity_reference.py | 82 +++++++++++++++++++++++ modules/field_simple.py | 24 +++---- modules/workbench_utils.py | 108 +++++++++++++++--------------- workbench | 91 +++---------------------- 4 files changed, 157 insertions(+), 148 deletions(-) create mode 100644 modules/field_entity_reference.py diff --git a/modules/field_entity_reference.py b/modules/field_entity_reference.py new file mode 100644 index 00000000..f2ecda52 --- /dev/null +++ b/modules/field_entity_reference.py @@ -0,0 +1,82 @@ +from workbench_utils import * + + +def update(config, field_definition, csv_row_value, node_field_values, custom_field_name, node_id): + if field_definition['field_type'] == 'entity_reference': + if field_definition['target_type'] == 'taxonomy_term': + target_type = 'taxonomy_term' + field_vocabs = get_field_vocabularies(config, field_definition) + if config['subdelimiter'] in csv_row_value: + prepared_tids = [] + delimited_values = csv_row_value.split(config['subdelimiter']) + for delimited_value in delimited_values: + tid = prepare_term_id(config, field_vocabs, delimited_value) + tid = str(tid) + prepared_tids.append(tid) + csv_row_value = config['subdelimiter'].join(prepared_tids) + else: + csv_row_value = prepare_term_id(config, field_vocabs, csv_row_value) + csv_row_value = str(csv_row_value) + + if field_definition['target_type'] == 'node': + target_type = 'node_type' + + if field_definition['cardinality'] == 1: + subvalues = csv_row_value.split(config['subdelimiter']) + custom_field_json = [{'target_id': subvalues[0], 'target_type': target_type}] + if len(subvalues) > 1: + log_field_cardinality_violation(custom_field_name, node_id, '1') + # Cardinality has a limit. + elif field_definition['cardinality'] > 1: + if config['update_mode'] == 'append': + # Append to existing values. + existing_target_ids = get_target_ids(node_field_values[custom_field_name]) + num_existing_values = len(existing_target_ids) + else: + existing_target_ids = [] + num_existing_values = 0 + + if config['subdelimiter'] in csv_row_value: + field_values = [] + subvalues = csv_row_value.split(config['subdelimiter']) + for subvalue in subvalues: + if subvalue in existing_target_ids: + existing_target_ids.remove(subvalue) + num_values_to_add = field_definition['cardinality'] - num_existing_values + subvalues = subvalues[:num_values_to_add] + if len(subvalues) > 0: + logging.warning("Adding all values in CSV field %s for node %s would exceed maximum number of " + + "allowed values (%s), so only adding %s values.", custom_field_name, node_id, field_definition['cardinality'], num_values_to_add) + logging.info("Updating node %s with %s values from CSV record.", node_id, num_values_to_add) + for subvalue in subvalues: + field_values.append({'target_id': subvalue, 'target_type': target_type}) + if config['update_mode'] == 'append': + custom_field_json = node_field_values[custom_field_name] + field_values + else: + custom_field_json = field_values + else: + logging.info("Not updating field %s node for %s, provided values do not contain any new values for this field.", custom_field_name, node_id) + else: + if num_existing_values + 1 <= field_definition['cardinality']: + custom_field_json = node_field_values[custom_field_name] + [{'target_id': csv_row_value, 'target_type': 'taxonomy_term'}] + else: + logging.warning("Not updating field %s node for %s, adding provided value would exceed maxiumum number of allowed values.", custom_field_name, node_id) + # Cardinality is unlimited. + else: + # Append to existing values. + if config['subdelimiter'] in csv_row_value: + field_values = [] + subvalues = csv_row_value.split(config['subdelimiter']) + for subvalue in subvalues: + field_values.append({'target_id': subvalue, 'target_type': target_type}) + if config['update_mode'] == 'append': + custom_field_json = node_field_values[custom_field_name] + field_values + else: + custom_field_json = field_values + else: + if config['update_mode'] == 'append': + custom_field_json = node_field_values[custom_field_name] + [{'target_id': csv_row_value, 'target_type': 'taxonomy_term'}] + else: + custom_field_json = [{'target_id': csv_row_value, 'target_type': target_type}] + + return custom_field_json diff --git a/modules/field_simple.py b/modules/field_simple.py index c0b92ac6..0d58929c 100644 --- a/modules/field_simple.py +++ b/modules/field_simple.py @@ -1,31 +1,31 @@ from workbench_utils import * -def update(config, field_definition, csv_row_value, node_field_values, custom_field, node_id): +def update(config, field_definition, csv_row_value, node_field_values, custom_field_name, node_id): if field_definition['cardinality'] == 1: subvalues = csv_row_value.split(config['subdelimiter']) node_field_value = [{'value': subvalues[0]}] if len(subvalues) > 1: - log_field_cardinality_violation(custom_field, node_id, '1') + log_field_cardinality_violation(custom_field_name, node_id, '1') elif field_definition['cardinality'] > 1: # Append to existing values. if config['subdelimiter'] in csv_row_value: field_values = [] subvalues = csv_row_value.split(config['subdelimiter']) if len(subvalues) > field_definition['cardinality']: - log_field_cardinality_violation(custom_field, node_id, field_definition['cardinality']) + log_field_cardinality_violation(custom_field_name, node_id, field_definition['cardinality']) subvalues = subvalues[:field_definition['cardinality']] for subvalue in subvalues: field_values.append({'value': subvalue}) if config['update_mode'] == 'append': - custom_field = node_field_values[custom_field] + field_values + custom_field_json = node_field_values[custom_field_name] + field_values else: - custom_field = field_values + custom_field_json = field_values else: if config['update_mode'] == 'append': - custom_field = node_field_values[custom_field] + [{'value': csv_row_value}] + custom_field_json = node_field_values[custom_field_name] + [{'value': csv_row_value}] else: - custom_field = [{'value': csv_row_value}] + custom_field_json = [{'value': csv_row_value}] # Cardinatlity is unlimited. else: # Append to existing values. @@ -35,13 +35,13 @@ def update(config, field_definition, csv_row_value, node_field_values, custom_fi for subvalue in subvalues: field_values.append({'value': subvalue}) if config['update_mode'] == 'append': - custom_field = node_field_values[custom_field] + field_values + custom_field_json = node_field_values[custom_field_name] + field_values else: - custom_field = field_values + custom_field_json = field_values else: if config['update_mode'] == 'append': - custom_field = node_field_values[custom_field] + [{'value': csv_row_value}] + custom_field_json = node_field_values[custom_field_name] + [{'value': csv_row_value}] else: - custom_field = [{'value': csv_row_value}] + custom_field_json = [{'value': csv_row_value}] - return custom_field + return custom_field_json diff --git a/modules/workbench_utils.py b/modules/workbench_utils.py index 34265795..13d19b3d 100644 --- a/modules/workbench_utils.py +++ b/modules/workbench_utils.py @@ -63,7 +63,6 @@ def set_config_defaults(args): if config['task'] == 'create': if 'published' not in config: config['published'] = 1 - if config['task'] == 'create': if 'preprocessors' in config_data: config['preprocessors'] = {} @@ -547,7 +546,7 @@ def check_input(config, args): if len(row['title']) > 255: message = "The 'title' column in row " + str(count) + " of your CSV file exceeds Drupal's maximum length of 255 characters." logging.error(message) - sys.exit('Error: ' + message) + sys.exit('Error: ' + message) if 'node_id' in csv_column_headers: csv_column_headers.remove('node_id') for csv_column_header in csv_column_headers: @@ -948,51 +947,51 @@ def create_term(config, vocab_id, term_name): return False term = { - "vid": [ - { - "target_id": vocab_id, - "target_type": "taxonomy_vocabulary" - } - ], - "status": [ - { - "value": True - } - ], - "name": [ - { - "value": term_name - } - ], - "description": [ - { - "value": "", - "format": None - } - ], - "weight": [ - { - "value": 0 - } - ], - "parent": [ - { - "target_id": None - } - ], - "default_langcode": [ - { - "value": True - } - ], - "path": [ - { - "alias": None, - "pid": None, - "langcode": "en" - } - ] - } + "vid": [ + { + "target_id": vocab_id, + "target_type": "taxonomy_vocabulary" + } + ], + "status": [ + { + "value": True + } + ], + "name": [ + { + "value": term_name + } + ], + "description": [ + { + "value": "", + "format": None + } + ], + "weight": [ + { + "value": 0 + } + ], + "parent": [ + { + "target_id": None + } + ], + "default_langcode": [ + { + "value": True + } + ], + "path": [ + { + "alias": None, + "pid": None, + "langcode": "en" + } + ] + } term_endpoint = config['host'] + '/taxonomy/term?_format=json' headers = { @@ -1042,11 +1041,11 @@ def prepare_term_id(config, vocab_ids, term): return tid -def get_field_vocabularies(config, field_definitions, field_name): +def get_field_vocabularies(config, field_definition): """Gets IDs of vocabularies linked from the current field (could be more than one). """ - if 'vocabularies' in field_definitions[field_name]: - vocabularies = field_definitions[field_name]['vocabularies'] + if 'vocabularies' in field_definition: + vocabularies = field_definition['vocabularies'] return vocabularies else: return False @@ -1104,7 +1103,10 @@ def validate_csv_field_cardinality(config, field_definitions, csv_data): for field_name in field_cardinalities.keys(): if field_name in row: delimited_field_values = row[field_name].split(config['subdelimiter']) - message = 'CSV field "' + field_name + '" in record with ID ' + row[config['id_field']] + ' contains more values than the number ' + if config['task'] == 'create': + message = 'CSV field "' + field_name + '" in record with ID ' + row[config['id_field']] + ' contains more values than the number ' + if config['task'] == 'update': + message = 'CSV field "' + field_name + '" in record with ID ' + row['node_id'] + ' contains more values than the number ' if field_cardinalities[field_name] == 1 and len(delimited_field_values) > 1: message_2 = 'allowed for that field (' + str(field_cardinalities[field_name]) + '). Workbench will add only the first value.' print('Warning: ' + message + message_2) @@ -1128,7 +1130,7 @@ def validate_taxonomy_field_values(config, field_definitions, csv_data): fields_with_vocabularies = dict() if column_name in field_definitions: if 'vocabularies' in field_definitions[column_name]: - vocabularies = get_field_vocabularies(config, field_definitions, column_name) + vocabularies = get_field_vocabularies(config, field_definitions[column_name]) all_tids_for_field = [] for vocabulary in vocabularies: terms = get_term_pairs(config, vocabulary) @@ -1149,7 +1151,7 @@ def validate_taxonomy_field_values(config, field_definitions, csv_data): new_term_names_in_csv = False for count, row in enumerate(csv_data, start=1): for column_name in fields_with_vocabularies: - this_fields_vocabularies = get_field_vocabularies(config, field_definitions, column_name) + this_fields_vocabularies = get_field_vocabularies(config, field_definitions[column_name]) this_fields_vocabularies_string = ', '.join(this_fields_vocabularies) if len(row[column_name]): # Allow for multiple values in one field. diff --git a/workbench b/workbench index 0e4a893a..46a8f97b 100755 --- a/workbench +++ b/workbench @@ -57,7 +57,7 @@ def create(): if 'parent_id' in row.keys() and row['parent_id'] in node_ids: row['field_member_of'] = node_ids[row['parent_id']] - # Add custom (non-required) CSV fields. + # Add custom (not required by Drupal or Workbench) CSV fields. required_fields = ['file', config['id_field'], 'title'] custom_fields = list( set(csv_column_headers) - set(required_fields)) @@ -101,7 +101,7 @@ def create(): if field_definitions[custom_field]['field_type'] == 'entity_reference': if field_definitions[custom_field]['target_type'] == 'taxonomy_term': target_type = 'taxonomy_term' - field_vocabs = get_field_vocabularies(config, field_definitions, custom_field) + field_vocabs = get_field_vocabularies(config, field_definitions[custom_field]) if config['subdelimiter'] in row[custom_field]: prepared_tids = [] delimited_values = row[custom_field].split(config['subdelimiter']) @@ -323,7 +323,7 @@ def update(): else: node_field_values = {} - # Add custom (non-required) fields. + # Add custom (non required by Drupal) fields. required_fields = ['node_id'] custom_fields = list( set(csv_column_headers) - set(required_fields)) @@ -334,87 +334,12 @@ def update(): # Entity reference fields: for taxonomy terms, target_type is 'taxonomy_term'; # for nodes, it's 'node_type'. + # During work on #47, we hard code logic to choose field, but ulitimately we want the choice + # of which field module to load be dynamic based on field_type, i.e., with module name + # being 'field_' + field_type value. if field_definitions[custom_field]['field_type'] == 'entity_reference': - if field_definitions[custom_field]['target_type'] == 'taxonomy_term': - target_type = 'taxonomy_term' - field_vocabs = get_field_vocabularies(config, field_definitions, custom_field) - if config['subdelimiter'] in row[custom_field]: - prepared_tids = [] - delimited_values = row[custom_field].split(config['subdelimiter']) - for delimited_value in delimited_values: - tid = prepare_term_id(config, field_vocabs, delimited_value) - tid = str(tid) - prepared_tids.append(tid) - row[custom_field] = config['subdelimiter'].join(prepared_tids) - else: - row[custom_field] = prepare_term_id(config, field_vocabs, row[custom_field]) - row[custom_field] = str(row[custom_field]) - - if field_definitions[custom_field]['target_type'] == 'node': - target_type = 'node_type' - - if field_definitions[custom_field]['cardinality'] == 1: - subvalues = row[custom_field].split(config['subdelimiter']) - node[custom_field] = [ - {'target_id': subvalues[0], 'target_type': target_type}] - if len(subvalues) > 1: - log_field_cardinality_violation(custom_field, row['node_id'], '1') - # Cardinality has a limit. - elif field_definitions[custom_field]['cardinality'] > 1: - if config['update_mode'] == 'append': - # Append to existing values. - existing_target_ids = get_target_ids(node_field_values[custom_field]) - num_existing_values = len(existing_target_ids) - else: - existing_target_ids = [] - num_existing_values = 0 - - if config['subdelimiter'] in row[custom_field]: - field_values = [] - subvalues = row[custom_field].split(config['subdelimiter']) - for subvalue in subvalues: - if subvalue in existing_target_ids: - existing_target_ids.remove(subvalue) - num_values_to_add = field_definitions[custom_field]['cardinality'] - num_existing_values - subvalues = subvalues[:num_values_to_add] - if len(subvalues) > 0: - logging.warning("Adding all values in CSV field %s for node %s would exceed maximum number of " + - "allowed values (%s), so only adding %s values.", custom_field, row['node_id'], field_definitions[custom_field]['cardinality'], num_values_to_add) - logging.info("Updating node %s with %s values from CSV record.", row['node_id'], num_values_to_add) - for subvalue in subvalues: - field_values.append({'target_id': subvalue, 'target_type': target_type}) - if config['update_mode'] == 'append': - node[custom_field] = node_field_values[custom_field] + field_values - else: - node[custom_field] = field_values - else: - logging.info("Not updating field %s node for %s, provided values do not contain any new values for this field.", custom_field, row['node_id']) - else: - if num_existing_values + 1 <= field_definitions[custom_field]['cardinality']: - node[custom_field] = node_field_values[custom_field] + [ - {'target_id': row[custom_field], - 'target_type': 'taxonomy_term'}] - else: - logging.warning("Not updating field %s node for %s, adding provided value would exceed maxiumum number of allowed values.", custom_field, row['node_id']) - # Cardinality is unlimited. - else: - # Append to existing values. - if config['subdelimiter'] in row[custom_field]: - field_values = [] - subvalues = row[custom_field].split(config['subdelimiter']) - for subvalue in subvalues: - field_values.append({'target_id': subvalue, 'target_type': target_type}) - if config['update_mode'] == 'append': - node[custom_field] = node_field_values[custom_field] + field_values - else: - node[custom_field] = field_values - else: - if config['update_mode'] == 'append': - node[custom_field] = node_field_values[custom_field] + [ - {'target_id': row[custom_field], - 'target_type': 'taxonomy_term'}] - else: - node[custom_field] = [{'target_id': row[custom_field], 'target_type': 'taxonomy_term'}] + entity_reference_field = importlib.import_module('field_entity_reference') + node[custom_field] = entity_reference_field.update(config, field_definitions[custom_field], row[custom_field], node_field_values, custom_field, row['node_id']) # Typed relation fields. elif field_definitions[custom_field]['field_type'] == 'typed_relation':