Skip to content

Commit

Permalink
WIP on #47.
Browse files Browse the repository at this point in the history
  • Loading branch information
@mjordan
mjordan committed May 5, 2020
1 parent 30657d3 commit 583afbe
Show file tree
Hide file tree
Showing 4 changed files with 157 additions and 148 deletions.
82 changes: 82 additions & 0 deletions modules/field_entity_reference.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
from workbench_utils import *


def update(config, field_definition, csv_row_value, node_field_values, custom_field_name, node_id):
if field_definition['field_type'] == 'entity_reference':
if field_definition['target_type'] == 'taxonomy_term':
target_type = 'taxonomy_term'
field_vocabs = get_field_vocabularies(config, field_definition)
if config['subdelimiter'] in csv_row_value:
prepared_tids = []
delimited_values = csv_row_value.split(config['subdelimiter'])
for delimited_value in delimited_values:
tid = prepare_term_id(config, field_vocabs, delimited_value)
tid = str(tid)
prepared_tids.append(tid)
csv_row_value = config['subdelimiter'].join(prepared_tids)
else:
csv_row_value = prepare_term_id(config, field_vocabs, csv_row_value)
csv_row_value = str(csv_row_value)

if field_definition['target_type'] == 'node':
target_type = 'node_type'

if field_definition['cardinality'] == 1:
subvalues = csv_row_value.split(config['subdelimiter'])
custom_field_json = [{'target_id': subvalues[0], 'target_type': target_type}]
if len(subvalues) > 1:
log_field_cardinality_violation(custom_field_name, node_id, '1')
# Cardinality has a limit.
elif field_definition['cardinality'] > 1:
if config['update_mode'] == 'append':
# Append to existing values.
existing_target_ids = get_target_ids(node_field_values[custom_field_name])
num_existing_values = len(existing_target_ids)
else:
existing_target_ids = []
num_existing_values = 0

if config['subdelimiter'] in csv_row_value:
field_values = []
subvalues = csv_row_value.split(config['subdelimiter'])
for subvalue in subvalues:
if subvalue in existing_target_ids:
existing_target_ids.remove(subvalue)
num_values_to_add = field_definition['cardinality'] - num_existing_values
subvalues = subvalues[:num_values_to_add]
if len(subvalues) > 0:
logging.warning("Adding all values in CSV field %s for node %s would exceed maximum number of " +
"allowed values (%s), so only adding %s values.", custom_field_name, node_id, field_definition['cardinality'], num_values_to_add)
logging.info("Updating node %s with %s values from CSV record.", node_id, num_values_to_add)
for subvalue in subvalues:
field_values.append({'target_id': subvalue, 'target_type': target_type})
if config['update_mode'] == 'append':
custom_field_json = node_field_values[custom_field_name] + field_values
else:
custom_field_json = field_values
else:
logging.info("Not updating field %s node for %s, provided values do not contain any new values for this field.", custom_field_name, node_id)
else:
if num_existing_values + 1 <= field_definition['cardinality']:
custom_field_json = node_field_values[custom_field_name] + [{'target_id': csv_row_value, 'target_type': 'taxonomy_term'}]
else:
logging.warning("Not updating field %s node for %s, adding provided value would exceed maxiumum number of allowed values.", custom_field_name, node_id)
# Cardinality is unlimited.
else:
# Append to existing values.
if config['subdelimiter'] in csv_row_value:
field_values = []
subvalues = csv_row_value.split(config['subdelimiter'])
for subvalue in subvalues:
field_values.append({'target_id': subvalue, 'target_type': target_type})
if config['update_mode'] == 'append':
custom_field_json = node_field_values[custom_field_name] + field_values
else:
custom_field_json = field_values
else:
if config['update_mode'] == 'append':
custom_field_json = node_field_values[custom_field_name] + [{'target_id': csv_row_value, 'target_type': 'taxonomy_term'}]
else:
custom_field_json = [{'target_id': csv_row_value, 'target_type': target_type}]

return custom_field_json
24 changes: 12 additions & 12 deletions modules/field_simple.py
View file
Original file line number Diff line number Diff line change
@@ -1,31 +1,31 @@
from workbench_utils import *


def update(config, field_definition, csv_row_value, node_field_values, custom_field, node_id):
def update(config, field_definition, csv_row_value, node_field_values, custom_field_name, node_id):
if field_definition['cardinality'] == 1:
subvalues = csv_row_value.split(config['subdelimiter'])
node_field_value = [{'value': subvalues[0]}]
if len(subvalues) > 1:
log_field_cardinality_violation(custom_field, node_id, '1')
log_field_cardinality_violation(custom_field_name, node_id, '1')
elif field_definition['cardinality'] > 1:
# Append to existing values.
if config['subdelimiter'] in csv_row_value:
field_values = []
subvalues = csv_row_value.split(config['subdelimiter'])
if len(subvalues) > field_definition['cardinality']:
log_field_cardinality_violation(custom_field, node_id, field_definition['cardinality'])
log_field_cardinality_violation(custom_field_name, node_id, field_definition['cardinality'])
subvalues = subvalues[:field_definition['cardinality']]
for subvalue in subvalues:
field_values.append({'value': subvalue})
if config['update_mode'] == 'append':
custom_field = node_field_values[custom_field] + field_values
custom_field_json = node_field_values[custom_field_name] + field_values
else:
custom_field = field_values
custom_field_json = field_values
else:
if config['update_mode'] == 'append':
custom_field = node_field_values[custom_field] + [{'value': csv_row_value}]
custom_field_json = node_field_values[custom_field_name] + [{'value': csv_row_value}]
else:
custom_field = [{'value': csv_row_value}]
custom_field_json = [{'value': csv_row_value}]
# Cardinatlity is unlimited.
else:
# Append to existing values.
Expand All @@ -35,13 +35,13 @@ def update(config, field_definition, csv_row_value, node_field_values, custom_fi
for subvalue in subvalues:
field_values.append({'value': subvalue})
if config['update_mode'] == 'append':
custom_field = node_field_values[custom_field] + field_values
custom_field_json = node_field_values[custom_field_name] + field_values
else:
custom_field = field_values
custom_field_json = field_values
else:
if config['update_mode'] == 'append':
custom_field = node_field_values[custom_field] + [{'value': csv_row_value}]
custom_field_json = node_field_values[custom_field_name] + [{'value': csv_row_value}]
else:
custom_field = [{'value': csv_row_value}]
custom_field_json = [{'value': csv_row_value}]

return custom_field
return custom_field_json
108 changes: 55 additions & 53 deletions modules/workbench_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@ def set_config_defaults(args):
if config['task'] == 'create':
if 'published' not in config:
config['published'] = 1

if config['task'] == 'create':
if 'preprocessors' in config_data:
config['preprocessors'] = {}
Expand Down Expand Up @@ -547,7 +546,7 @@ def check_input(config, args):
if len(row['title']) > 255:
message = "The 'title' column in row " + str(count) + " of your CSV file exceeds Drupal's maximum length of 255 characters."
logging.error(message)
sys.exit('Error: ' + message)
sys.exit('Error: ' + message)
if 'node_id' in csv_column_headers:
csv_column_headers.remove('node_id')
for csv_column_header in csv_column_headers:
Expand Down Expand Up @@ -948,51 +947,51 @@ def create_term(config, vocab_id, term_name):
return False

term = {
"vid": [
{
"target_id": vocab_id,
"target_type": "taxonomy_vocabulary"
}
],
"status": [
{
"value": True
}
],
"name": [
{
"value": term_name
}
],
"description": [
{
"value": "",
"format": None
}
],
"weight": [
{
"value": 0
}
],
"parent": [
{
"target_id": None
}
],
"default_langcode": [
{
"value": True
}
],
"path": [
{
"alias": None,
"pid": None,
"langcode": "en"
}
]
}
"vid": [
{
"target_id": vocab_id,
"target_type": "taxonomy_vocabulary"
}
],
"status": [
{
"value": True
}
],
"name": [
{
"value": term_name
}
],
"description": [
{
"value": "",
"format": None
}
],
"weight": [
{
"value": 0
}
],
"parent": [
{
"target_id": None
}
],
"default_langcode": [
{
"value": True
}
],
"path": [
{
"alias": None,
"pid": None,
"langcode": "en"
}
]
}

term_endpoint = config['host'] + '/taxonomy/term?_format=json'
headers = {
Expand Down Expand Up @@ -1042,11 +1041,11 @@ def prepare_term_id(config, vocab_ids, term):
return tid


def get_field_vocabularies(config, field_definitions, field_name):
def get_field_vocabularies(config, field_definition):
"""Gets IDs of vocabularies linked from the current field (could be more than one).
"""
if 'vocabularies' in field_definitions[field_name]:
vocabularies = field_definitions[field_name]['vocabularies']
if 'vocabularies' in field_definition:
vocabularies = field_definition['vocabularies']
return vocabularies
else:
return False
Expand Down Expand Up @@ -1104,7 +1103,10 @@ def validate_csv_field_cardinality(config, field_definitions, csv_data):
for field_name in field_cardinalities.keys():
if field_name in row:
delimited_field_values = row[field_name].split(config['subdelimiter'])
message = 'CSV field "' + field_name + '" in record with ID ' + row[config['id_field']] + ' contains more values than the number '
if config['task'] == 'create':
message = 'CSV field "' + field_name + '" in record with ID ' + row[config['id_field']] + ' contains more values than the number '
if config['task'] == 'update':
message = 'CSV field "' + field_name + '" in record with ID ' + row['node_id'] + ' contains more values than the number '
if field_cardinalities[field_name] == 1 and len(delimited_field_values) > 1:
message_2 = 'allowed for that field (' + str(field_cardinalities[field_name]) + '). Workbench will add only the first value.'
print('Warning: ' + message + message_2)
Expand All @@ -1128,7 +1130,7 @@ def validate_taxonomy_field_values(config, field_definitions, csv_data):
fields_with_vocabularies = dict()
if column_name in field_definitions:
if 'vocabularies' in field_definitions[column_name]:
vocabularies = get_field_vocabularies(config, field_definitions, column_name)
vocabularies = get_field_vocabularies(config, field_definitions[column_name])
all_tids_for_field = []
for vocabulary in vocabularies:
terms = get_term_pairs(config, vocabulary)
Expand All @@ -1149,7 +1151,7 @@ def validate_taxonomy_field_values(config, field_definitions, csv_data):
new_term_names_in_csv = False
for count, row in enumerate(csv_data, start=1):
for column_name in fields_with_vocabularies:
this_fields_vocabularies = get_field_vocabularies(config, field_definitions, column_name)
this_fields_vocabularies = get_field_vocabularies(config, field_definitions[column_name])
this_fields_vocabularies_string = ', '.join(this_fields_vocabularies)
if len(row[column_name]):
# Allow for multiple values in one field.
Expand Down
Loading

0 comments on commit 583afbe

Please sign in to comment.