diff --git a/pull.py b/pull.py new file mode 100644 index 00000000..6589259d --- /dev/null +++ b/pull.py @@ -0,0 +1,96 @@ +from formpack.remote_pack import RemoteFormPack, FORMPACK_DATA_DIR + +import os +import json +import argparse +import importlib + + +parser = argparse.ArgumentParser(description='Initialize RemoteFormPack.') + +parser.add_argument('--refresh-data', + dest='refresh_data', + help='flush data cache', + action='store_true') + +parser.add_argument('--print-stats', + dest='print_stats', + help='print stats from the dataset', + action='store_true') + +parser.add_argument('--print-survey', + dest='print_survey', + help='print survey questions', + action='store_true') + +parser.add_argument('--print-submissions', + dest='print_submissions', + help='print submissions from the dataset', + action='store_true') + +parser.add_argument('--run-module', + dest='run_module', + help='import and run a module', + action='store') + +parser.add_argument('--account', + dest='account', + help='server:account corresponding to local config', + action='store') + +parser.add_argument('uid', + nargs=1, + help='formid', + action='store') + + +def load_pack(uid, account): + accounts_file = os.environ.get('KOBO_ACCOUNTS', False) + if not accounts_file: + accounts_file = os.path.join(FORMPACK_DATA_DIR, 'accounts.json') + if not os.path.exists(accounts_file): + raise ValueError('need an accounts json file in {}'.format( + accounts_file)) + with open(accounts_file, 'r') as ff: + accounts = json.loads(ff.read()) + try: + _account = accounts[account] + except KeyError: + raise ValueError('accounts.json needs a configuration for {}'.format( + account)) + return RemoteFormPack(uid=uid, + token=_account['token'], + api_url=_account['api_url'], + ) + + +def run(args): + rpack = load_pack(uid=args.uid[0], + account=args.account) + + if args.refresh_data: + print('clearing submissions') + rpack.clear_submissions() + + rpack.pull() + formpk = rpack.create_pack() + + if args.print_stats: + print(json.dumps(formpk._stats, indent=2)) + + if args.print_survey: + print(json.dumps(formpk.get_survey(), indent=2)) + + if args.print_submissions: + print(json.dumps(list(rpack.submissions), indent=2)) + + if args.run_module: + _mod = importlib.import_module(args.run_module) + if not hasattr(_mod, 'run') and hasattr(_mod.run, '__call__'): + _mod.run(formpk, submissions=rpack.submissions) + else: + raise ValueError('run-module parameter must be an importable ' + 'module with a method "run"') + +if __name__ == '__main__': + run(parser.parse_args()) diff --git a/requirements.txt b/requirements.txt index b26f8d14..6fb8a8ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ path.py==8.1.2 PyExcelerate==0.6.7 pyquery==1.2.11 pyxform==0.9.22 -statistics==1.0.3.5 \ No newline at end of file +requests==2.13.0 +statistics==1.0.3.5 diff --git a/src/formpack/pack.py b/src/formpack/pack.py index 5fb06629..519ee589 100644 --- a/src/formpack/pack.py +++ b/src/formpack/pack.py @@ -23,8 +23,6 @@ class FormPack(object): - - # TODO: make a clear signature for __init__ def __init__(self, versions=None, title='Submissions', id_string=None, default_version_id_key='__version__', strict_schema=False, @@ -47,14 +45,14 @@ def __init__(self, versions=None, title='Submissions', id_string=None, self.root_node_name = root_node_name self.title = title + self.strict_schema = strict_schema self.asset_type = asset_type - self.load_all_versions(versions) def __repr__(self): - return '' % self._stats() + return '' % self._stats_str def version_id_keys(self, _versions=None): # if no parameter is passed, default to 'all' @@ -67,6 +65,12 @@ def version_id_keys(self, _versions=None): _id_keys.append(_id_key) return _id_keys + @property + def latest_version(self): + if len(self.versions) > 0: + return self.versions.values()[-1] + else: + raise ValueError('No versions available.') @property def available_translations(self): @@ -93,15 +97,27 @@ def __getitem__(self, index): except IndexError: raise IndexError('version at index %d is not available' % index) + @property def _stats(self): _stats = OrderedDict() + _stats['title'] = self.title _stats['id_string'] = self.id_string - _stats['versions'] = len(self.versions) - # _stats['submissions'] = self.submissions_count() - _stats['row_count'] = len(self[-1].schema.get('content', {}) - .get('survey', [])) + _vs = self.versions.values() + if len(_vs) > 0: + _content = _vs[-1].schema.get('content', {}) + _survey = _content.get('survey', []) + _stats['row_count'] = len(_survey) + + _versions = [] + for (vid, version) in self.versions.items(): + _versions.append(version._stats()) + _stats['versions'] = _versions + return _stats + + @property + def _stats_str(self): # returns stats in the format [ key="value" ] - return '\n\t'.join('%s="%s"' % item for item in _stats.items()) + return '\n\t'.join('%s="%s"' % item for item in self._stats.items()) def load_all_versions(self, versions): for schema in versions: @@ -123,6 +139,8 @@ def load_version(self, schema): unique accross an entire FormPack. It can be None, but only for one version in the FormPack. """ + if 'content' not in schema: + raise ValueError('''"content" is not an available key: {}'''.format(schema.keys())) replace_aliases(schema['content'], in_place=True) expand_content(schema['content'], in_place=True) @@ -161,6 +179,14 @@ def load_version(self, schema): self.versions[form_version.id] = form_version + def _latest_change(self): + _lvs = len(self.versions) + _keys = self.versions.keys() + if _lvs > 1: + v1 = _keys[_lvs - 2] + v2 = _keys[_lvs - 1] + return self.version_diff(v1, v2) + def version_diff(self, vn1, vn2): v1 = self.versions[vn1] v2 = self.versions[vn2] @@ -266,6 +292,11 @@ def to_dict(self, **kwargs): out[u'asset_type'] = self.asset_type return out + def get_survey(self): + return [ + row for row in self.latest_version.rows(include_groups=True) + ] + def to_json(self, **kwargs): return json.dumps(self.to_dict(), **kwargs) diff --git a/src/formpack/remote_pack.py b/src/formpack/remote_pack.py new file mode 100644 index 00000000..d55c8c63 --- /dev/null +++ b/src/formpack/remote_pack.py @@ -0,0 +1,166 @@ +# coding: utf-8 + +from __future__ import (unicode_literals, print_function, + absolute_import, division) + +from .pack import FormPack + +import json +import errno +import requests +from urlparse import urlparse +from argparse import Namespace as Ns +from os import (path, makedirs, unlink) + +FORMPACK_DATA_DIR = path.join(path.expanduser('~'), + '.formpack') + + +def mkdir_p(_path): + try: + makedirs(_path) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and path.isdir(_path): + pass + else: + raise + + +class RemoteFormPack: + def __init__(self, uid, + token, + api_url, + data_dir=None): + self.uid = uid + self.api_token = token + self.api_url = api_url + self._data_dir = data_dir or FORMPACK_DATA_DIR + + self.data_dir = path.join(self._data_dir, self.uid) + mkdir_p(self.data_dir) + self.paths = { + 'versions/': self.path('versions'), + 'data': self.path('data.json'), + 'context': self.path('context.json'), + 'asset': self.path('asset.json'), + } + self._versions_dir = self.path('versions') + self._data_path = self.path('data.json') + self._context_path = self.path('context.json') + mkdir_p(self.path('versions')) + self._asset_url = '{}{}'.format(self.api_url, self.uid) + self.asset = Ns(**self._query_asset()) + self.context = Ns(**self._query_kcform()) + + def path(self, *args): + return path.join(self.data_dir, *args) + + def _query_asset(self): + if not path.exists(self.path('asset.json')): + ad = requests.get('{}/?format=json'.format(self._asset_url), + headers=self._headers(), + ).json() + if 'detail' in ad and ad['detail'] == 'Invalid token.': + raise ValueError('Invalid token. Is it the correct server?') + elif 'detail' in ad: + raise ValueError("Error querying API: {}".format( + ad['detail'])) + # content is ultimately pulled form the "version" file + del ad['content'] + with open(self.path('asset.json'), 'w') as ff: + ff.write(json.dumps(ad, indent=2)) + return ad + else: + with open(self.path('asset.json'), 'r') as ff: + return json.loads(ff.read()) + + def _query_kcform(self): + asset = self.asset + if not path.exists(self.path('context.json')): + _deployment_identifier = asset.deployment__identifier + _deployment = urlparse(_deployment_identifier) + ctx = { + 'kc_api_url': '{}://{}/api/v1'.format(_deployment.scheme, + _deployment.netloc), + } + _url = '{}/forms?id_string={}'.format(ctx['kc_api_url'], + self.uid) + r2 = requests.get(_url, headers=self._headers()).json() + ctx['kc_formid'] = r2[0]['formid'] + with open(self.path('context.json'), 'w') as ff: + ff.write(json.dumps(ctx, indent=2)) + return ctx + else: + with open(self.path('context.json'), 'r') as ff: + return json.loads(ff.read()) + + def pull(self): + if not path.exists(self.path('data.json')): + _data_url = '{}/data/{}?{}'.format(self.context.kc_api_url, + self.context.kc_formid, + 'format=json', + ) + _data = requests.get(_data_url, headers=self._headers()).json() + with open(self.path('data.json'), 'w') as ff: + ff.write(json.dumps(_data, indent=2)) + _version_ids = set([i['__version__'] for i in _data]) + for version_id in _version_ids: + self.load_version(version_id) + + def load_version(self, version_id): + _version_path = path.join(self.path('versions'), + '{}.json'.format(version_id) + ) + if not path.exists(_version_path): + _version_url = '{}/{}/{}/?format=json'.format( + self._asset_url, + 'versions', + version_id) + vd = requests.get(_version_url, headers=self._headers()).json() + if vd.get('detail') == 'Not found.': + raise Exception('Version not found') + with open(_version_path, 'w') as ff: + ff.write(json.dumps(vd, indent=2)) + return vd + else: + with open(_version_path, 'r') as ff: + return json.loads(ff.read()) + + def _headers(self, upd={}): + return dict({'Content-Type': 'application/json', + 'Authorization': 'Token {}'.format(self.api_token), + }, **upd) + + def create_pack(self): + if not path.exists(self._data_path): + raise Exception('cannot generate formpack without running ' + 'remote_pack.pull()') + _version_ids = set([s['__version__'] for s in self.submissions]) + self.versions = [] + for version_id in _version_ids: + _v = self.load_version(version_id) + _v['version'] = version_id + _v['date_deployed'] = _v.pop('date_deployed', None) + self.versions.append(_v) + return FormPack(versions=self.versions, id_string=self.uid, + title=self.asset.name, + ) + + def stats(self): + pck = self.create_pack() + _stats = pck._stats() + return _stats + + def _submissions(self): + with open(self.path('data.json'), 'r') as ff: + return json.loads(ff.read()) + + def clear_submissions(self): + _data_path = self.path('data.json') + if path.exists(_data_path): + unlink(_data_path) + + @property + def submissions(self): + for submission in self._submissions(): + yield submission diff --git a/src/formpack/schema/__init__.py b/src/formpack/schema/__init__.py index d7c756e8..3b5050bc 100644 --- a/src/formpack/schema/__init__.py +++ b/src/formpack/schema/__init__.py @@ -4,5 +4,79 @@ absolute_import, division) +from functools import partial + from .fields import * # noqa -from .datadef import * # noqa \ No newline at end of file +from .datadef import * # noqa +from ..translated_item import TranslatedItem + + +def _field_from_dict(definition, hierarchy=None, + section=None, field_choices={}, + translations=None): + """Return an instance of a Field class matching this JSON field def + + Depending of the data datype extracted from the field definition, + this method will return an instance of a different class. + + Args: + definition (dict): Description + group (FormGroup, optional): The group this field is into + section (FormSection, optional): The section this field is into + field_choices (dict, optional): + A mapping of all the FormChoice instances available for + this form. + + Returns: + Union[FormChoiceField, FormChoiceField, + FormChoiceFieldWithMultipleSelect, FormField]: + The FormField instance matching this definiton. + """ + name = definition.get('$autoname', definition.get('name')) + labels = definition.get('label') + if labels: + labels = TranslatedItem(labels, + translations=translations) + else: + labels = TranslatedItem() + + # normalize spaces + data_type = definition['type'] + choice = None + + if ' ' in data_type: + raise ValueError('invalid data_type: %s' % data_type) + + if data_type in ('select_one', 'select_multiple'): + choice_id = definition['select_from_list_name'] + choice = field_choices[choice_id] + + data_type_classes = { + "select_one": FormChoiceField, + "select_multiple": FormChoiceFieldWithMultipleSelect, + "geopoint": FormGPSField, + "date": DateField, + "text": TextField, + "barcode": TextField, + + # calculate is usually not text but for our purpose it's good + # enough + "calculate": TextField, + "acknowledge": TextField, + "integer": NumField, + 'decimal': NumField, + + # legacy type, treat them as text + "select_one_external": partial(TextField, data_type=data_type), + "cascading_select": partial(TextField, data_type=data_type), + } + + cls = data_type_classes.get(data_type, FormField) + return cls(name=name, + labels=labels, + data_type=data_type, + hierarchy=hierarchy, + section=section, + choice=choice, + src=definition, + ) diff --git a/src/formpack/schema/datadef.py b/src/formpack/schema/datadef.py index 73982605..cc89d747 100644 --- a/src/formpack/schema/datadef.py +++ b/src/formpack/schema/datadef.py @@ -16,16 +16,21 @@ from collections import OrderedDict from ..constants import UNTRANSLATED +from ..translated_item import TranslatedItem class FormDataDef(object): """ Any object composing a form. It's only used with a subclass. """ - def __init__(self, name, labels=None, has_stats=False, *args, **kwargs): + def __init__(self, name, labels=None, + has_stats=False, src=None, + *args, **kwargs): self.name = name - self.labels = labels or {} + # assert labels is None or isinstance(labels, TranslatedItem) + self.labels = labels or TranslatedItem() self.value_names = self.get_value_names() self.has_stats = has_stats + self.src = src def __repr__(self): return "<%s name='%s'>" % (self.__class__.__name__, self.name) @@ -33,25 +38,6 @@ def __repr__(self): def get_value_names(self): return [self.name] - @classmethod - def from_json_definition(cls, definition): - labels = cls._extract_json_labels(definition) - return cls(definition['name'], labels) - - @classmethod - def _extract_json_labels(cls, definition): - """ Extract translation labels from the JSON data definition """ - labels = OrderedDict() - if "label" in definition: - labels[UNTRANSLATED] = definition['label'] - - for key, val in definition.items(): - if key.startswith('label:'): - # sometime the label can be separated with 2 :: - _, lang = re.split(r'::?', key, maxsplit=1, flags=re.U) - labels[lang] = val - return labels - class FormGroup(FormDataDef): # useful to get __repr__ pass @@ -65,21 +51,29 @@ def __init__(self, name="submissions", labels=None, fields=None, *args, **kwargs): if labels is None: - labels = {UNTRANSLATED: 'submissions'} + labels = TranslatedItem() + self.parent = parent super(FormSection, self).__init__(name, labels, *args, **kwargs) self.fields = fields or OrderedDict() - self.parent = parent self.children = list(children) self.hierarchy = list(hierarchy) + [self] # do not include the root section in the path self.path = '/'.join(info.name for info in self.hierarchy[1:]) - @classmethod - def from_json_definition(cls, definition, hierarchy=(None,), parent=None): - labels = cls._extract_json_labels(definition) - return cls(definition['name'], labels, hierarchy=hierarchy, parent=parent) + @property + def rows(self, include_groups=False): + for (name, field) in self.fields.items(): + if include_groups and hasattr(self, 'begin_rows'): + for row in self.begin_rows: + yield row.src + + yield field.src + + if include_groups and hasattr(self, 'end_rows'): + for row in self.end_rows: + yield row.src def get_label(self, lang=UNTRANSLATED): return [self.labels.get(lang) or self.name] @@ -91,38 +85,9 @@ def __repr__(self): class FormChoice(FormDataDef): def __init__(self, name, *args, **kwargs): - super(FormChoice, self).__init__(name, *args, **kwargs) self.name = name - self.options = OrderedDict() - - @classmethod - def all_from_json_definition(cls, definition, translation_list): - all_choices = {} - for choice_definition in definition: - choice_name = choice_definition.get('name') - choice_key = choice_definition.get('list_name') - if not choice_name or not choice_key: - continue - - if choice_key not in all_choices: - all_choices[choice_key] = FormChoice(choice_key) - choices = all_choices[choice_key] - - option = choices.options[choice_name] = {} - - # apparently choices dont need a label if they have an image - if 'label' in choice_definition: - _label = choice_definition['label'] - else: - _label = choice_definition.get('image') - if isinstance(_label, basestring): - _label = [_label] - elif _label is None and len(translation_list) == 1: - _label = [None] - option['labels'] = OrderedDict(zip(translation_list, _label)) - option['name'] = choice_name - return all_choices - + self.options = kwargs.pop('options', OrderedDict()) + super(FormChoice, self).__init__(name, *args, **kwargs) @property def translations(self): diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index 9657a587..e77d7c83 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -6,7 +6,6 @@ import re from operator import itemgetter -from functools import partial try: xrange = xrange @@ -33,7 +32,7 @@ class FormField(FormDataDef): def __init__(self, name, labels, data_type, hierarchy=None, section=None, can_format=True, has_stats=None, *args, **kwargs): - + # assert not isinstance(labels, dict): self.data_type = data_type self.section = section self.can_format = can_format @@ -106,76 +105,6 @@ def __repr__(self): args = (self.__class__.__name__, self.name, self.data_type) return "<%s name='%s' type='%s'>" % args - @classmethod - def from_json_definition(cls, definition, hierarchy=None, - section=None, field_choices={}, - translations=None): - """Return an instance of a Field class matching this JSON field def - - Depending of the data datype extracted from the field definition, - this method will return an instance of a different class. - - Args: - definition (dict): Description - group (FormGroup, optional): The group this field is into - section (FormSection, optional): The section this field is into - field_choices (dict, optional): - A mapping of all the FormChoice instances available for - this form. - - Returns: - Union[FormChoiceField, FormChoiceField, - FormChoiceFieldWithMultipleSelect, FormField]: - The FormField instance matching this definiton. - """ - name = definition['name'] - label = definition.get('label') - if label: - labels = OrderedDict(zip(translations, label)) - else: - labels = {} - - # normalize spaces - data_type = definition['type'] - choice = None - - if ' ' in data_type: - raise ValueError('invalid data_type: %s' % data_type) - - if data_type in ('select_one', 'select_multiple'): - choice_id = definition['select_from_list_name'] - choice = field_choices[choice_id] - - data_type_classes = { - "select_one": FormChoiceField, - "select_multiple": FormChoiceFieldWithMultipleSelect, - "geopoint": FormGPSField, - "date": DateField, - "text": TextField, - "barcode": TextField, - - # calculate is usually not text but for our purpose it's good - # enough - "calculate": TextField, - "acknowledge": TextField, - "integer": NumField, - 'decimal': NumField, - - # legacy type, treat them as text - "select_one_external": partial(TextField, data_type=data_type), - "cascading_select": partial(TextField, data_type=data_type), - } - - args = { - 'name': name, - 'labels': labels, - 'data_type': data_type, - 'hierarchy': hierarchy, - 'section': section, - 'choice': choice - } - return data_type_classes.get(data_type, cls)(**args) - def format(self, val, lang=UNSPECIFIED_TRANSLATION, context=None): return {self.name: val} diff --git a/src/formpack/translated_item.py b/src/formpack/translated_item.py new file mode 100644 index 00000000..451ae4b6 --- /dev/null +++ b/src/formpack/translated_item.py @@ -0,0 +1,35 @@ +# coding: utf-8 + +from __future__ import (unicode_literals, print_function, absolute_import, + division) + +from collections import OrderedDict +from .errors import TranslationError + + +class TranslatedItem(object): + def __init__(self, values=[], translations=[], strict=False, context=''): + if isinstance(values, OrderedDict): + translations = values.keys() + values = values.values() + + if len(translations) == 1 and translations[0] is None and \ + len(values) == 0: + values = [None] + if len(values) > len(translations): + raise TranslationError('String count exceeds translation count. {}' + .format(context)) + if strict and len(values) < len(translations): + raise TranslationError('Translation count does not match' + ' string count. {}'.format(context)) + else: + while len(values) < len(translations): + values = values + [None] + + self._translations = OrderedDict(zip(translations, values)) + + def __getitem__(self, index): + return self._translations.values()[index] + + def get(self, key, default=None): + return self._translations.get(key, default) diff --git a/src/formpack/utils/xform_tools.py b/src/formpack/utils/xform_tools.py index 2da4c4b5..b33dac47 100644 --- a/src/formpack/utils/xform_tools.py +++ b/src/formpack/utils/xform_tools.py @@ -11,6 +11,7 @@ from pyquery import PyQuery +# made obsolete by replace_aliases DATA_TYPE_ALIASES = ( ("add select one prompt using", 'select_one'), ("select one from", 'select_one'), @@ -20,7 +21,7 @@ ("select all that apply from", 'select_multiple'), ("select multiple", 'select_multiple'), ("select all that apply", 'select_multiple'), - ("select_one_external", "select one external"), + ("select one external", "select_one_external"), ('cascading select', 'cascading_select'), ('location', 'geopoint'), ("begin lgroup", 'begin_repeat'), @@ -79,8 +80,12 @@ def parse_xml_to_data(xml_str): def normalize_data_type(data_type): - """ Normalize spaces and aliases for field data types """ + """ + Normalize spaces and aliases for field data types + note: this method is made obsolete by the pre-processing + "replace_aliases" step. + """ # normalize spaces data_type = ' '.join(data_type.split()) diff --git a/src/formpack/version.py b/src/formpack/version.py index 62ac6c4b..ccd5c478 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -18,27 +18,47 @@ from .errors import SchemaError from .utils.flatten_content import flatten_content from .schema import (FormField, FormGroup, FormSection, FormChoice) -from .errors import TranslationError +from .translated_item import TranslatedItem +from .schema import _field_from_dict -class LabelStruct(object): - ''' - LabelStruct stores labels + translations assigned to `field.labels` - ''' +def get_labels(choice_definition, translation_list): + # choices dont need a label if they have an image + if 'label' in choice_definition: + _label = choice_definition['label'] + elif 'image' in choice_definition: + _label = choice_definition['image'] + else: + _label = None - def __init__(self, labels=[], translations=[]): - if len(labels) != len(translations): - errmsg = 'Mismatched labels and translations: [{}] [{}] ' \ - '{}!={}'.format(', '.join(labels), - ', '.join(translations), len(labels), - len(translations)) - raise TranslationError(errmsg) - self._labels = labels - self._translations = translations - self._vals = dict(zip(translations, labels)) + if isinstance(_label, basestring): + _label = [_label] + elif _label is None and len(translation_list) == 1: + _label = [None] - def get(self, key, default=None): - return self._vals.get(key, default) + return OrderedDict(zip(translation_list, _label)) + + +def choices_from_structures(definition, translation_list): + all_choices = {} + for choice_definition in definition: + choice_name = choice_definition.get('$autovalue', + choice_definition.get('name')) + choice_key = choice_definition.get('list_name') + if not choice_name or not choice_key: + continue + + if choice_key not in all_choices: + all_choices[choice_key] = { + 'name': choice_key, + 'options': OrderedDict(), + } + + all_choices[choice_key]['options'][choice_name] = { + 'labels': get_labels(choice_definition, translation_list), + 'name': choice_name, + } + return all_choices.items() class FormVersion(object): @@ -50,10 +70,7 @@ def verify_schema_structure(cls, struct): raise SchemaError('version content must have "survey"') validate_content(struct['content']) - # QUESTION FOR ALEX: get rid off _root_node_name ? What is it for ? def __init__(self, form_pack, schema): - - # QUESTION FOR ALEX: why this check ? if 'name' in schema: raise ValueError('FormVersion should not have a name parameter. ' 'consider using "title" or "id_string"') @@ -65,6 +82,8 @@ def __init__(self, form_pack, schema): # form version id, unique to this version of the form self.id = schema.get('version') + self.date = schema.get('date_deployed') + self.version_id_key = schema.get('version_id_key', form_pack.default_version_id_key) @@ -98,7 +117,12 @@ def __init__(self, form_pack, schema): # TODO: put those parts in a separate method and unit test it survey = content.get('survey', []) - fields_by_name = dict(map(lambda row: (row.get('name'), row), survey)) + + fields_by_name = dict(map(lambda row: + (row.get('$autoname', row.get('name')), + row, + ), + survey)) # Analyze the survey schema and extract the informations we need # to build the export: the sections, the choices, the fields @@ -108,12 +132,17 @@ def __init__(self, form_pack, schema): # Choices are the list of values you can choose from to answer a # specific question. They can have translatable labels. choices_definition = content.get('choices', ()) - field_choices = FormChoice.all_from_json_definition(choices_definition, - self.translations) + + field_choices = dict([ + (key, FormChoice(key, options=itm['options'])) + for (key, itm) in choices_from_structures(choices_definition, + list(self.translations)) + ]) # Extract fields data group = None - section = FormSection(name=form_pack.title) + section = FormSection(name=form_pack.title, src=False) + self._main_section = section self.sections[form_pack.title] = section # Those will keep track of were we are while traversing the @@ -131,6 +160,8 @@ def __init__(self, form_pack, schema): continue data_type = normalize_data_type(data_type) + if '$autoname' in data_definition: + data_definition['name'] = data_definition.get('$autoname') name = data_definition.get('name') # parse closing groups and repeat @@ -146,8 +177,8 @@ def __init__(self, form_pack, schema): continue if data_type == 'end_repeat': - # We go up in one level of nesting, so we set the current section - # to be what used to be the parent section + # We go up in one level of nesting, so we set the current + # section to be what used to be the parent section hierarchy.pop() section = section_stack.pop() continue @@ -159,7 +190,14 @@ def __init__(self, form_pack, schema): if data_type == 'begin_group': group_stack.append(group) - group = FormGroup.from_json_definition(data_definition) + + labels = TranslatedItem(data_definition.get('label', []), + translations=self.translations, + ) + + group = FormGroup(data_definition['name'], labels, + src=data_definition) + # We go down in one level on nesting, so save the parent group. # Parent maybe None, in that case we are at the top level. hierarchy.append(group) @@ -170,9 +208,17 @@ def __init__(self, form_pack, schema): # Parent maybe None, in that case we are at the top level. parent_section = section - section = FormSection.from_json_definition(data_definition, - hierarchy, - parent=parent_section) + labels = TranslatedItem(data_definition.get('label', []), + translations=self.translations, + ) + _repeat_name = data_definition.get('$autoname', data_definition.get('name')) + section = FormSection(_repeat_name, + labels, + hierarchy=hierarchy, + src=data_definition, + parent=parent_section, + ) + self.sections[section.name] = section hierarchy.append(section) section_stack.append(parent_section) @@ -181,33 +227,42 @@ def __init__(self, form_pack, schema): # If we are here, it's a regular field # Get the the data name and type - field = FormField.from_json_definition(data_definition, - hierarchy, section, - field_choices, - translations=self.translations) + field = _field_from_dict(data_definition, + hierarchy, section, + field_choices, + translations=self.translations) section.fields[field.name] = field _f = fields_by_name[field.name] - _labels = LabelStruct() + _labels = TranslatedItem() if 'label' in _f: if not isinstance(_f['label'], list): _f['label'] = [_f['label']] - _labels = LabelStruct(labels=_f['label'], - translations=self.translations) + _labels = TranslatedItem(_f['label'], + translations=self.translations) field.labels = _labels assert 'labels' not in _f def __repr__(self): - return '' % self._stats() + return '' % self._stats_str() + + def rows(self, include_groups=False): + for row in self._main_section.rows: + yield row def _stats(self): _stats = OrderedDict() _stats['id_string'] = self._get_id_string() _stats['version'] = self.id + _stats['date'] = self.date _stats['row_count'] = len(self.schema.get('content', {}).get('survey', [])) # returns stats in the format [ key="value" ] + return _stats + + def _stats_str(self): + _stats = self._stats() return '\n\t'.join(map(lambda key: '%s="%s"' % (key, str(_stats[key])), _stats.keys())) diff --git a/tests/fixtures/remote_pack/README.rst b/tests/fixtures/remote_pack/README.rst new file mode 100644 index 00000000..7b21255e --- /dev/null +++ b/tests/fixtures/remote_pack/README.rst @@ -0,0 +1,7 @@ +This directory provides an `__init__.py` file which can be put in the same +directory as a backup of production data via the remote_pack.py CLI utility + +Simply copy or move the directory containing data (example: "~/.formpack/aFoRmId654321") +to the parent directory "tests/fixtures" and copy the accompanying `__init__.py` +file. The data can then be tested with formpack through the "build_fixtures(...)" +method. diff --git a/tests/fixtures/remote_pack/__init__.py b/tests/fixtures/remote_pack/__init__.py new file mode 100644 index 00000000..b89a866e --- /dev/null +++ b/tests/fixtures/remote_pack/__init__.py @@ -0,0 +1,48 @@ +# coding: utf-8 + +from __future__ import (unicode_literals, print_function, + absolute_import, division) + +''' +This file provides a way to open production data and +write a test based on that data. +''' +import os +import glob +import json +from collections import defaultdict + + +DIR = os.path.dirname(os.path.abspath(__file__)) + + +def _path(*args): + return os.path.join(DIR, *args) + +def _load_version_file(vf): + with open(vf, 'r') as version_f: + vx = json.loads(version_f.read()) + uid = vx.pop('uid') + vx.update({ + 'version': uid, + 'submissions': submissions.get(uid, []) + }) + return vx + +with open(_path('asset.json'), 'r') as asset_file: + asset = json.loads(asset_file.read()) + +with open(_path('data.json'), 'r') as submission_f: + submissions = defaultdict(list) + for submission in json.loads(submission_f.read()): + vkey = submission['__version__'] + submissions[vkey].append(submission) + +_version_file_list = glob.glob(_path('versions', '*')) + + +DATA = { + 'title': asset['name'], + 'versions': [_load_version_file(vf) for vf in _version_file_list], + 'submissions': submissions, +} diff --git a/tests/test_translated_item.py b/tests/test_translated_item.py new file mode 100644 index 00000000..3e952db7 --- /dev/null +++ b/tests/test_translated_item.py @@ -0,0 +1,53 @@ +# coding: utf-8 + +from __future__ import (unicode_literals, print_function, + absolute_import, division) +import json +import pytest +from collections import OrderedDict + +from formpack.translated_item import TranslatedItem +from formpack.errors import TranslationError + + +def test_simple_translated(): + t1 = TranslatedItem(['x', 'y'], + translations=['langx', 'langy']) + expected = '{"langx": "x", "langy": "y"}' + assert json.dumps(t1._translations) == expected + + # an OrderedDict can also be used to initialize a TranslatedItem + t2 = TranslatedItem(OrderedDict([ + ('langx', 'x'), + ('langy', 'y'), + ])) + assert json.dumps(t1._translations) == json.dumps(t2._translations) + + +def test_invalid_translateds(): + with pytest.raises(TranslationError): + TranslatedItem(['two', 'translations'], + translations=['onelang']) + + ti = TranslatedItem(['one translation'], + # strict=False, by default + translations=['lang1', 'lang2']) + assert ti._translations['lang2'] is None + + # this happens when a null translation is created on an + # incomplete form. If we were to make this invalid, we should + # enforce it at a different step + ti = TranslatedItem([], translations=[None]) + assert json.dumps(ti._translations) == '{"null": null}' + + +def test_strict_translateds(): + with pytest.raises(TranslationError): + TranslatedItem(['one translation'], + strict=True, + translations=['two', 'langs']) + + with pytest.raises(TranslationError): + TranslatedItem(['two', 'translations'], + translations=['onelang'], + strict=True)