- AIncremental checkin, most unit tests appear to be working

ckan/lib/navl/dictization_functions.py

@@ -138,11 +138,19 @@ def augment_data(data, schema):
     new_data = copy.copy(data)
 
     ## fill junk and extras
-
+    #FIXME this part of the code is unclear, needs more unit tests and comments
     for key, value in new_data.items():
         if key in full_schema:
             continue
 
+        #Hack: this part of the function moves things into __extras or __junk.
+        #if for any reason, the password1 / password2 fields don't both have
+        #validators assigned, they'll end up getting moved, which causes validation
+        #to fail. for these, we have a special case to keep them as their own key:value
+        #pairs and not move them.
+        if key in [('password1', ),('password2', ), ('password',)]:
+            continue
+
         ## check if any thing naugthy is placed against subschemas
         initial_tuple = key[::2]
         if initial_tuple in [initial_key[:len(initial_tuple)]
@@ -151,7 +159,9 @@ def augment_data(data, schema):
                 raise DataError('Only lists of dicts can be placed against '
                                 'subschema %s, not %s' % (key,type(data[key])))
 
-        if key[:-1] in key_combinations:
+        # This section appears to move data which does not have validators/converters assigned (is not in full_schema)
+        # to the '__extras' or '__junk' fields
+        if  key[:-1] in key_combinations:
             extras_key = key[:-1] + ('__extras',)
             extras = new_data.get(extras_key, {})
             extras[key[-1]] = value
@@ -171,9 +181,6 @@ def augment_data(data, schema):
     return new_data
 
 def convert(converter, key, converted_data, errors, context):
-    #TODO remove debug statement
-    log.debug('convert called, field to validate: {0}'.format(key))
-
     value = converted_data.get(key, '')
 
     if inspect.isclass(converter) and issubclass(converter, fe.Validator):

ckan/logic/validators.py

@@ -16,6 +16,10 @@
 
 from ckan.common import _
 
+import logging
+
+log = logging.getLogger(__name__)
+
 Invalid = df.Invalid
 StopOnError = df.StopOnError
 Missing = df.Missing
@@ -583,16 +587,18 @@ def user_name_validator(key, data, errors, context):
             # existing user's name to that name.
             errors[key].append(_('That login name is not available.'))
 
-def user_both_passwords_entered(key, data, errors, context):
+def user_both_passwords_entered(key, data, errors, context=None):
 
     password1 = data.get(('password1',),None)
     password2 = data.get(('password2',),None)
 
-    if password1 is None or password1 == '' or \
-       password2 is None or password2 == '':
-        errors[('password',)].append(_('Please enter both passwords'))
+    if not (password1 and password2):
+        if key in errors:
+            errors[key].append(_('Please enter both passwords'))
+        else:
+            errors[key] = [_('Please enter both passwords')]
 
-def user_password_validator(key, data, errors, context):
+def user_password_validator(key, data, errors, context=None):
     value = data[key]
 
     if isinstance(value, Missing):
@@ -604,18 +610,21 @@ def user_password_validator(key, data, errors, context):
     elif len(value) < 4:
         errors[('password',)].append(_('Your password must be 4 characters or longer'))
 
-def user_passwords_match(key, data, errors, context):
+def user_passwords_match(key, data, errors, context=None):
 
     password1 = data.get(('password1',),None)
     password2 = data.get(('password2',),None)
 
     if not password1 == password2:
-        errors[key].append(_('The passwords you entered do not match'))
+        if key in errors:
+            errors[key].append(_('The passwords you entered do not match'))
+        else:
+            errors[key] = [_('The passwords you entered do not match')]
     else:
         #Set correct password
         data[('password',)] = password1
 
-def user_password_not_empty(key, data, errors, context):
+def user_password_not_empty(key, data, errors, context=None):
     '''Only check if password is present if the user is created via action API.
        If not, user_both_passwords_entered will handle the validation'''
 
@@ -630,7 +639,7 @@ def user_password_not_empty(key, data, errors, context):
             errors[key]=[_('Missing value')]
             raise StopOnError
 
-def user_about_validator(value,context):
+def user_about_validator(value,context=None):
     if 'http://' in value or 'https://' in value:
         raise Invalid(_('Edit not allowed as it looks like spam. Please avoid links in your description.'))
 
@@ -738,7 +747,7 @@ def role_exists(role, context):
 
 
 def datasets_with_no_organization_cannot_be_private(key, data, errors,
-        context):
+        context=None):
 
     dataset_id = data.get(('id',))
     owner_org = data.get(('owner_org',))

ckan/tests/legacy/lib/test_navl.py

@@ -118,7 +118,7 @@ def test_make_full_schema():
                                                               ('4', 1, '30')])
 
 
-def test_augment_junk_and_extras():
+def test_augment_data_basic():
 
     assert augment_data(data, schema) == {
          ('__junk',): {('4', 1, '30'): '30 value 1'},
@@ -137,7 +137,7 @@ def test_augment_junk_and_extras():
 
 def test_identity_validation():
 
-    
+
     converted_data, errors = validate_flattened(data, schema)
     print errors
     print converted_data
@@ -267,6 +267,25 @@ def test_flatten():
 
     assert data == unflatten(flatten_dict(data))
 
+def test_flatten_empty():
+
+    data = {}
+    flattened_data = flatten_dict(data)
+
+    assert flattened_data == {}
+
+    assert data == unflatten(flattened_data)
+
+def test_flatten_no_extra():
+
+    data = {'foo':'bar',
+            'foo2':'bar2'}
+    flattened_data = flatten_dict(data)
+
+    assert flattened_data == {('foo',): u'bar', ('foo2', ): u'bar2'}
+
+    assert data == unflatten(flattened_data)
+
 
 def test_simple():
     schema = {
@@ -397,9 +416,13 @@ def test_missing_data_field_ignore_missing():
     assert not errors
 
 def test_formencode_url_validator_handled():
+    """
+    There were some errors being raised when using the formencode library's URL validator. Verifying that said
+    validator no longer causes an error in CKAN
+    """
     data = {'foo':u'bar'}
 
-    schema = {'field_not_found_in_data': [validators.URL(max=350), ignore_missing]
+    schema = {'field_not_found_in_data': [validators.URL, ignore_missing]
     }