Merge pull request #212 from rdmorganiser/madmp

Refactor project exports

rdmo/conditions/templates/conditions/conditions.html

@@ -63,12 +63,12 @@ <h2>{% trans 'Export' %}</h2>
         </li>
     </ul>
 
-
     <h2>{% trans 'Import' %}</h2>
     <ul class="list-unstyled">
-      <li>
-         {% include 'core/import_form.html' with import_url_name="conditions_import" filetype='xml'%}
-      </li>
+        <li>
+            {% url 'conditions_import' 'xml' as upload_url %}
+            {% include 'core/upload_form.html' with upload_url=upload_url %}
+        </li>
     </ul>
 
 {% endblock %}

rdmo/conditions/views.py

@@ -3,19 +3,19 @@
 from django.conf import settings
 from django.http import HttpResponse, HttpResponseRedirect
 from django.shortcuts import render
-from django.utils.translation import ugettext_lazy as _
-from django.views.generic import TemplateView, ListView
 from django.urls import reverse_lazy
+from django.utils.translation import ugettext_lazy as _
+from django.views.generic import ListView, TemplateView
 
 from rdmo.core.exports import prettify_xml
 from rdmo.core.imports import handle_uploaded_file, read_xml_file
 from rdmo.core.utils import get_model_field_meta, render_to_format
-from rdmo.core.views import ModelPermissionMixin, CSRFViewMixin
+from rdmo.core.views import CSRFViewMixin, ModelPermissionMixin
 
 from .imports import import_conditions
 from .models import Condition
-from .serializers.export import ConditionSerializer as ExportSerializer
 from .renderers import XMLRenderer
+from .serializers.export import ConditionSerializer as ExportSerializer
 
 log = logging.getLogger(__name__)
 
@@ -53,7 +53,6 @@ def render_to_response(self, context, **response_kwargs):
 class ConditionsImportXMLView(ModelPermissionMixin, ListView):
     permission_required = ('conditions.add_condition', 'conditions.change_condition', 'conditions.delete_condition')
     success_url = reverse_lazy('conditions')
-    parsing_error_template = 'core/import_parsing_error.html'
 
     def get(self, request, *args, **kwargs):
         return HttpResponseRedirect(self.success_url)
@@ -69,7 +68,10 @@ def post(self, request, *args, **kwargs):
         tree = read_xml_file(tempfilename)
         if tree is None:
             log.info('Xml parsing error. Import failed.')
-            return render(request, self.parsing_error_template, status=400)
+            return render(request, 'core/error.html', {
+                'title': _('Import error'),
+                'error': _('The content of the xml file does not consist of well formed data or markup.')
+            }, status=400)
         else:
             import_conditions(tree)
             return HttpResponseRedirect(self.success_url)

rdmo/core/imports.py

@@ -61,6 +61,12 @@ def handle_uploaded_file(filedata):
     return tempfilename
 
 
+def open_uploaded_file(tempfilename):
+    tempfilename = generate_tempfile_name()
+    return open(tempfilename)
+
+
+# TODO should be moved to xml.py
 def read_xml_file(filename):
     tree = None
     try:

rdmo/core/management/commands/import.py

@@ -1,14 +1,12 @@
 import logging
-
 import xml.etree.ElementTree as et
 
-from django.contrib.auth.models import User
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand
 
 from rdmo.conditions.imports import import_conditions
 from rdmo.domain.imports import import_domain
 from rdmo.options.imports import import_options
-from rdmo.projects.imports import import_project
+# from rdmo.projects.imports import import_project
 from rdmo.questions.imports import import_questions
 from rdmo.tasks.imports import import_tasks
 from rdmo.views.imports import import_views
@@ -34,10 +32,3 @@ def handle(self, *args, **options):
             import_questions(root)
             import_tasks(root)
             import_views(root)
-        elif root.tag == 'project':
-            try:
-                user = User.objects.get(username=options['user'])
-            except User.DoesNotExist:
-                raise CommandError('Give a valid username using --user.')
-
-            import_project(user, root)

rdmo/core/models.py

@@ -1,9 +1,7 @@
 from django.db import models
 from django.utils.timezone import now
-from django.utils.translation import ugettext_lazy as _
 from django.utils.translation import get_language
-
-from rdmo.core.exceptions import RDMOException
+from django.utils.translation import ugettext_lazy as _
 from rdmo.core.utils import get_languages
 
 
@@ -16,10 +14,11 @@ class Meta:
         abstract = True
 
     def save(self, *args, **kwargs):
-        if not self.id or kwargs.get('force_insert', False):
+        if self.created is None:
             self.created = now()
 
         self.updated = now()
+
         super(Model, self).save(*args, **kwargs)
 
 

rdmo/core/settings.py

@@ -182,7 +182,10 @@
     'PROFILE_UPDATE',
     'PROFILE_DELETE',
     'SHIBBOLETH',
-    'MULTISITE'
+    'MULTISITE',
+    'EXPORT_FORMATS',
+    'PROJECT_EXPORTS',
+    'PROJECT_IMPORTS'
 ]
 
 EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
@@ -199,6 +202,16 @@
     ('tex', _('LaTeX'))
 )
 
+PROJECT_EXPORTS = [
+    ('xml', _('RDMO XML'), 'rdmo.projects.exports.RDMOXMLExport'),
+    ('csvcomma', _('CSV comma separated'), 'rdmo.projects.exports.CSVCommaExport'),
+    ('csvsemicolon', _('CSV semicolon separated'), 'rdmo.projects.exports.CSVSemicolonExport')
+]
+
+PROJECT_IMPORTS = [
+    ('xml', _('RDMO XML'), 'rdmo.projects.imports.RDMOXMLImport'),
+]
+
 DEFAULT_URI_PREFIX = 'http://example.com/terms'
 
 VENDOR_CDN = True

rdmo/core/static/core/css/base.scss

@@ -71,6 +71,12 @@ code {
     word-wrap: break-word;
 }
 
+.table-break-word {
+    td {
+        word-break: break-all;
+    }
+}
+
 /* navbar */
 
 .navbar-default {
@@ -321,3 +327,12 @@ li.has-warning > a.control-label > i {
 .select2-results__option--highlighted{
     background-color: $link-color !important,
 }
+
+.import-submit {
+    margin-top: 70px;
+    width: 100%;
+}
+
+.import-checkbox {
+    text-align: center;
+}

rdmo/core/static/core/css/upload_form.scss → rdmo/core/static/core/css/upload-form.scss

@@ -9,7 +9,7 @@ form.upload-form {
         border-radius: 4px;
         margin-bottom: 10px;
         margin-right: 5px;
-        
+
         p,
         input {
             width: 141px;

rdmo/core/templates/core/error.html

@@ -0,0 +1,12 @@
+{% extends 'core/page.html' %}
+{% load i18n %}
+
+{% block page %}
+
+    <h1>{{ title }}</h1>
+
+    {% for error in errors %}
+    <p>{{ error }}</p>
+    {% endfor %}
+
+{% endblock %}

rdmo/core/templates/core/import_form.html → rdmo/core/templates/core/upload_form.html

@@ -4,14 +4,14 @@
 {% load core_tags %}
 
 {% compress css %}
-<link rel="stylesheet" type="text/x-scss" href="{% static 'core/css/upload_form.scss' %}" />
+<link rel="stylesheet" type="text/x-scss" href="{% static 'core/css/upload-form.scss' %}" />
 {% endcompress %}
 
-<form class="upload-form" action="{% url import_url_name filetype %}" method="POST" enctype="multipart/form-data">
+<form class="upload-form" action="{{ upload_url }}" method="POST" enctype="multipart/form-data">
     {% csrf_token %}
     <div class="upload-form-field pull-left">
-        <input type="file" name="uploaded_file" accept=".xml" />
-        <p>{% trans "Select xml file" %}</p>
+        <input type="file" name="uploaded_file" />
+        <p>{% trans "Select file" %}</p>
     </div>
 
     <button type="submit" name="submit-btn" class="btn btn-primary">{% trans "Upload" %}</button>
@@ -22,7 +22,7 @@
         $('button[name="submit-btn"]').prop('disabled', true)
         $('form input').change(function() {
             if (this.files[0] == null) {
-                $('form p').text('{% trans "Select xml file" %}')
+                $('form p').text('{% trans "Select file" %}')
                 $('button[name="submit-btn"]').prop('disabled', true)
             } else {
                 $('form p').text(this.files[0].name);

rdmo/core/utils.py

@@ -1,5 +1,5 @@
 import csv
-import json
+import importlib
 import logging
 import os
 import re
@@ -209,3 +209,8 @@ def sanitize_url(s):
         else:
             s = re.sub('/+', '/', s)
     return s
+
+
+def import_class(string):
+    module_name, class_name = string.rsplit('.', 1)
+    return getattr(importlib.import_module(module_name), class_name)

rdmo/core/xml.py

@@ -1,10 +1,18 @@
 import logging
 import re
+
 import defusedxml.ElementTree as ET
 
 log = logging.getLogger(__name__)
 
 
+def read_xml_file(file_name):
+    try:
+        return ET.parse(file_name).getroot()
+    except Exception as e:
+        log.error('Xml parsing error: ' + str(e))
+
+
 def flat_xml_to_elements(treenode):
     elements = []
     ns_map = get_ns_map(treenode)
@@ -46,9 +54,11 @@ def get_ns_tag(tag, ns_map):
 def get_ns_map(treenode):
     ns_map = {}
     treestring = ET.tostring(treenode, encoding='utf8', method='xml')
-    match = re.search(r'(xmlns:)(.*?)(=")(.*?)(")', str(treestring))
-    if bool(match) is True:
-        ns_map = {match.group(2): match.group(4)}
+
+    for match in re.finditer(r'(xmlns:)(.*?)(=")(.*?)(")', str(treestring)):
+        if match:
+            ns_map[match.group(2)] = match.group(4)
+
     return ns_map
 
 

rdmo/domain/models.py

@@ -1,6 +1,5 @@
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
-
 from mptt.models import MPTTModel, TreeForeignKey
 
 from rdmo.core.utils import get_uri_prefix

rdmo/domain/templates/domain/domain.html

@@ -78,9 +78,10 @@ <h2>{% trans 'Export' %}</h2>
 
     <h2>{% trans 'Import' %}</h2>
     <ul class="list-unstyled">
-      <li>
-         {% include 'core/import_form.html' with import_url_name="domain_import" filetype='xml'%}
-      </li>
+        <li>
+            {% url 'domain_import' 'xml' as upload_url %}
+            {% include 'core/upload_form.html' with upload_url=upload_url %}
+        </li>
     </ul>
 
 {% endblock %}