From 1eb032ae76a73d8a48133c3d9c5546830c24ed94 Mon Sep 17 00:00:00 2001 From: kollivier Date: Sun, 12 Aug 2018 14:33:18 -0700 Subject: [PATCH 01/28] objects.create automatically calls save, so no need to call it again afterwards. --- contentcuration/contentcuration/tests/testdata.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/contentcuration/contentcuration/tests/testdata.py b/contentcuration/contentcuration/tests/testdata.py index 8c1e952e0a..a0367fd1ea 100644 --- a/contentcuration/contentcuration/tests/testdata.py +++ b/contentcuration/contentcuration/tests/testdata.py @@ -120,7 +120,6 @@ def node(data, parent=None): # Create topics if data['kind_id'] == "topic": new_node = cc.ContentNode(kind=topic(), parent=parent, title=data['title'], node_id=data['node_id']) - new_node.save() for child in data['children']: child_node = node(child, parent=new_node) @@ -128,7 +127,6 @@ def node(data, parent=None): # Create videos elif data['kind_id'] == "video": new_node = cc.ContentNode(kind=video(), parent=parent, title=data['title'], node_id=data['node_id'], license=license_wtfpl()) - new_node.save() video_file = fileobj_video(contents="Video File").next() video_file.contentnode = new_node video_file.preset_id = format_presets.VIDEO_HIGH_RES @@ -138,7 +136,6 @@ def node(data, parent=None): elif data['kind_id'] == "exercise": extra_fields = "{{\"mastery_model\":\"{}\",\"randomize\":true,\"m\":{},\"n\":{}}}".format(data['mastery_model'], data.get('m') or 0, data.get('n') or 0) new_node = cc.ContentNode(kind=exercise(), parent=parent, title=data['title'], node_id=data['node_id'], license=license_wtfpl(), extra_fields=extra_fields) - new_node.save() for assessment_item in data['assessment_items']: mixer.blend(cc.AssessmentItem, contentnode=new_node, From b124a0dd6a62e584a408db7ae1dd5717ffc39e2c Mon Sep 17 00:00:00 2001 From: kollivier Date: Tue, 14 Aug 2018 14:14:17 -0700 Subject: [PATCH 02/28] Revert removed calls to save as we are creating models directly rather than using objects.create. --- contentcuration/contentcuration/tests/testdata.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contentcuration/contentcuration/tests/testdata.py b/contentcuration/contentcuration/tests/testdata.py index a0367fd1ea..8c1e952e0a 100644 --- a/contentcuration/contentcuration/tests/testdata.py +++ b/contentcuration/contentcuration/tests/testdata.py @@ -120,6 +120,7 @@ def node(data, parent=None): # Create topics if data['kind_id'] == "topic": new_node = cc.ContentNode(kind=topic(), parent=parent, title=data['title'], node_id=data['node_id']) + new_node.save() for child in data['children']: child_node = node(child, parent=new_node) @@ -127,6 +128,7 @@ def node(data, parent=None): # Create videos elif data['kind_id'] == "video": new_node = cc.ContentNode(kind=video(), parent=parent, title=data['title'], node_id=data['node_id'], license=license_wtfpl()) + new_node.save() video_file = fileobj_video(contents="Video File").next() video_file.contentnode = new_node video_file.preset_id = format_presets.VIDEO_HIGH_RES @@ -136,6 +138,7 @@ def node(data, parent=None): elif data['kind_id'] == "exercise": extra_fields = "{{\"mastery_model\":\"{}\",\"randomize\":true,\"m\":{},\"n\":{}}}".format(data['mastery_model'], data.get('m') or 0, data.get('n') or 0) new_node = cc.ContentNode(kind=exercise(), parent=parent, title=data['title'], node_id=data['node_id'], license=license_wtfpl(), extra_fields=extra_fields) + new_node.save() for assessment_item in data['assessment_items']: mixer.blend(cc.AssessmentItem, contentnode=new_node, From 1fa69041cfa8e347885625a7f2c4938d3a00dab5 Mon Sep 17 00:00:00 2001 From: kollivier Date: Sun, 12 Aug 2018 14:33:18 -0700 Subject: [PATCH 03/28] objects.create automatically calls save, so no need to call it again afterwards. --- contentcuration/contentcuration/tests/testdata.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/contentcuration/contentcuration/tests/testdata.py b/contentcuration/contentcuration/tests/testdata.py index 8c1e952e0a..a0367fd1ea 100644 --- a/contentcuration/contentcuration/tests/testdata.py +++ b/contentcuration/contentcuration/tests/testdata.py @@ -120,7 +120,6 @@ def node(data, parent=None): # Create topics if data['kind_id'] == "topic": new_node = cc.ContentNode(kind=topic(), parent=parent, title=data['title'], node_id=data['node_id']) - new_node.save() for child in data['children']: child_node = node(child, parent=new_node) @@ -128,7 +127,6 @@ def node(data, parent=None): # Create videos elif data['kind_id'] == "video": new_node = cc.ContentNode(kind=video(), parent=parent, title=data['title'], node_id=data['node_id'], license=license_wtfpl()) - new_node.save() video_file = fileobj_video(contents="Video File").next() video_file.contentnode = new_node video_file.preset_id = format_presets.VIDEO_HIGH_RES @@ -138,7 +136,6 @@ def node(data, parent=None): elif data['kind_id'] == "exercise": extra_fields = "{{\"mastery_model\":\"{}\",\"randomize\":true,\"m\":{},\"n\":{}}}".format(data['mastery_model'], data.get('m') or 0, data.get('n') or 0) new_node = cc.ContentNode(kind=exercise(), parent=parent, title=data['title'], node_id=data['node_id'], license=license_wtfpl(), extra_fields=extra_fields) - new_node.save() for assessment_item in data['assessment_items']: mixer.blend(cc.AssessmentItem, contentnode=new_node, From 6d7b9ba0a18610e76f54d4c88ddc0c2ecf212907 Mon Sep 17 00:00:00 2001 From: kollivier Date: Tue, 14 Aug 2018 14:14:17 -0700 Subject: [PATCH 04/28] Revert removed calls to save as we are creating models directly rather than using objects.create. --- contentcuration/contentcuration/tests/testdata.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contentcuration/contentcuration/tests/testdata.py b/contentcuration/contentcuration/tests/testdata.py index a0367fd1ea..8c1e952e0a 100644 --- a/contentcuration/contentcuration/tests/testdata.py +++ b/contentcuration/contentcuration/tests/testdata.py @@ -120,6 +120,7 @@ def node(data, parent=None): # Create topics if data['kind_id'] == "topic": new_node = cc.ContentNode(kind=topic(), parent=parent, title=data['title'], node_id=data['node_id']) + new_node.save() for child in data['children']: child_node = node(child, parent=new_node) @@ -127,6 +128,7 @@ def node(data, parent=None): # Create videos elif data['kind_id'] == "video": new_node = cc.ContentNode(kind=video(), parent=parent, title=data['title'], node_id=data['node_id'], license=license_wtfpl()) + new_node.save() video_file = fileobj_video(contents="Video File").next() video_file.contentnode = new_node video_file.preset_id = format_presets.VIDEO_HIGH_RES @@ -136,6 +138,7 @@ def node(data, parent=None): elif data['kind_id'] == "exercise": extra_fields = "{{\"mastery_model\":\"{}\",\"randomize\":true,\"m\":{},\"n\":{}}}".format(data['mastery_model'], data.get('m') or 0, data.get('n') or 0) new_node = cc.ContentNode(kind=exercise(), parent=parent, title=data['title'], node_id=data['node_id'], license=license_wtfpl(), extra_fields=extra_fields) + new_node.save() for assessment_item in data['assessment_items']: mixer.blend(cc.AssessmentItem, contentnode=new_node, From 390cc63b120aa2ee7617ae0edce3fadedf14a811 Mon Sep 17 00:00:00 2001 From: kollivier Date: Thu, 12 Jul 2018 11:44:06 -0700 Subject: [PATCH 05/28] Apply Micah's changes and also fix some new issues uncovered by tests to get the code to the same state as it was before revert. --- contentcuration/contentcuration/decorators.py | 8 ++++---- .../contentcuration/migrations/0001_initial.py | 3 --- contentcuration/contentcuration/templates/base.html | 2 +- .../contentcuration/templates/registration/login.html | 2 +- .../contentcuration/templates/registration/logout.html | 2 +- .../templates/registration/registration_form.html | 2 +- .../registration/registration_information_form.html | 2 +- .../kolibri_content/migrations/0001_initial.py | 3 --- 8 files changed, 9 insertions(+), 15 deletions(-) diff --git a/contentcuration/contentcuration/decorators.py b/contentcuration/contentcuration/decorators.py index 8ac5e4030a..e046c8991b 100644 --- a/contentcuration/contentcuration/decorators.py +++ b/contentcuration/contentcuration/decorators.py @@ -2,7 +2,7 @@ from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from django.core.urlresolvers import reverse_lazy -from django.shortcuts import render_to_response, redirect +from django.shortcuts import render, redirect from django.template import RequestContext from contentcuration.models import Channel from contentcuration.utils.policies import check_policies @@ -18,7 +18,7 @@ def wrap(request, *args, **kwargs): if expected_agent in user_agent_string: return function(request, *args, **kwargs) - return render_to_response('unsupported_browser.html', context_instance=RequestContext(request)) + return render(request, 'unsupported_browser.html') wrap.__doc__ = function.__doc__ wrap.__name__ = function.__name__ @@ -41,7 +41,7 @@ def wrap(request, *args, **kwargs): try: channel = Channel.objects.get(pk=kwargs['channel_id']) except ObjectDoesNotExist: - return render_to_response('channel_not_found.html', context_instance=RequestContext(request)) + return render(request, 'channel_not_found.html') if channel.public or \ channel.editors.filter(id=request.user.id).exists() or \ @@ -65,7 +65,7 @@ def wrap(request, *args, **kwargs): return function(request, *args, **kwargs) except ObjectDoesNotExist: - return render_to_response('channel_not_found.html', context_instance=RequestContext(request)) + return render(request, 'channel_not_found.html') wrap.__doc__ = function.__doc__ wrap.__name__ = function.__name__ diff --git a/contentcuration/contentcuration/migrations/0001_initial.py b/contentcuration/contentcuration/migrations/0001_initial.py index 55a1132168..7ae7100c19 100644 --- a/contentcuration/contentcuration/migrations/0001_initial.py +++ b/contentcuration/contentcuration/migrations/0001_initial.py @@ -91,9 +91,6 @@ class Migration(migrations.Migration): 'verbose_name': 'Topic', 'verbose_name_plural': 'Topics', }, - managers=[ - ('_default_manager', django.db.models.manager.Manager()), - ], ), migrations.CreateModel( name='ContentTag', diff --git a/contentcuration/contentcuration/templates/base.html b/contentcuration/contentcuration/templates/base.html index 308df5f6fb..ced3c87157 100644 --- a/contentcuration/contentcuration/templates/base.html +++ b/contentcuration/contentcuration/templates/base.html @@ -98,7 +98,7 @@ {% if user.is_admin %}
  • {% trans "Administration" %}
    • {% endif %}
  • {% trans "Settings" %}
  • {% trans "Help" %}
    • -
  • {% trans "Log Out" %}
    • +
  • {% trans "Log Out" %}
    • diff --git a/contentcuration/contentcuration/templates/registration/login.html b/contentcuration/contentcuration/templates/registration/login.html index ee85e20efe..6e9e785be3 100644 --- a/contentcuration/contentcuration/templates/registration/login.html +++ b/contentcuration/contentcuration/templates/registration/login.html @@ -23,7 +23,7 @@

    {% trans 'Log In' %}

    - diff --git a/contentcuration/contentcuration/templates/registration/registration_information_form.html b/contentcuration/contentcuration/templates/registration/registration_information_form.html index 7aa4b3498d..84deeeeeb8 100644 --- a/contentcuration/contentcuration/templates/registration/registration_information_form.html +++ b/contentcuration/contentcuration/templates/registration/registration_information_form.html @@ -90,7 +90,7 @@

    {% trans 'Create an Account' %}

    - +
    diff --git a/contentcuration/kolibri_content/migrations/0001_initial.py b/contentcuration/kolibri_content/migrations/0001_initial.py index ade8346393..ccba39f563 100644 --- a/contentcuration/kolibri_content/migrations/0001_initial.py +++ b/contentcuration/kolibri_content/migrations/0001_initial.py @@ -69,9 +69,6 @@ class Migration(migrations.Migration): options={ 'ordering': ('lft',), }, - managers=[ - ('_default_manager', django.db.models.manager.Manager()), - ], ), migrations.CreateModel( name='ContentTag', From 7307bbf06559f9dcd4ae7c1ab7c8bdfc76e38653 Mon Sep 17 00:00:00 2001 From: kollivier Date: Fri, 13 Jul 2018 10:05:49 -0700 Subject: [PATCH 06/28] Fix login template name. --- contentcuration/contentcuration/urls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contentcuration/contentcuration/urls.py b/contentcuration/contentcuration/urls.py index 5cb0739747..62d86d39c3 100644 --- a/contentcuration/contentcuration/urls.py +++ b/contentcuration/contentcuration/urls.py @@ -237,8 +237,8 @@ def get_queryset(self): # Add account/registration endpoints urlpatterns += [ - url(r'^accounts/login/$', auth_views.login, {'template_name': 'registration/login.html', 'authentication_form': LoginForm}, name='auth_login'), - url(r'^accounts/logout/$', auth_views.logout, {'template_name': 'registration/logout.html'}), + url(r'^accounts/login/$', auth_views.login, {'template_name': 'registration/login.html'}, 'authentication_form': LoginForm, name='auth_login'), + url(r'^accounts/logout/$', auth_views.logout, {'template_name': 'registration/logout.html'}, name='logout'), url( r'^accounts/password/reset/$', registration_views.custom_password_reset, From 655ef38aac1fe88ac2b5a2d88a40ad7a26ee9cc6 Mon Sep 17 00:00:00 2001 From: kollivier Date: Tue, 24 Jul 2018 14:05:36 -0700 Subject: [PATCH 07/28] Actually commit the updated dependency versions to requirements.txt... --- requirements.txt | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..be65b8840b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,48 @@ +Django==1.11 +django-filter==0.9.2 +django-mptt==0.9.0 +django-pg-utils==0.1.5 +djangorestframework==3.8.2 +pdfkit==0.6.1 +psycopg2-binary==2.7.4 +Markdown==2.6.2 +Pillow==5.0.0 +psutil==3.1.1 +django-js-reverse==0.6.1 +django-registration==2.1.2 +django-hashedfilenamestorage==1.0.0 +djangorestframework-bulk==0.2.1 +django-email-extras==0.3.3 +kolibri==0.7.0 +validators +le-utils>=0.1.11 +gunicorn==19.6.0 +django-mailgun==0.9.1 +pressurecooker==0.0.15 +metaphone==0.6 +porter2stemmer==1.0 +django-postmark==0.1.6 +ddtrace==0.8.0 +jsonfield==2.0.2 +newrelic==2.86.3.70 +celery==4.1.1 +redis==2.10.5 +reportlab==3.4.0 +python-resize-image==1.1.11 +pycountry==17.5.14 +sphinx==1.6.4 +sphinx-autobuild==0.7.1 +sphinx_rtd_theme +sphinx-intl +django-s3-storage==0.12.4 +boto3==1.6.5 +requests==2.19.1 +minio==3.0.3 +pathlib +sqlalchemy==1.2.2 +xhtml2pdf==0.2.1 +progressbar2==3.38.0 +python-postmark==0.5.0 +django-webpack-loader==0.6.0 +gspread==3.0.1 +google-api-python-client==1.6.4 From e1a8d01d8bee334635a88c4773bd64831c889db0 Mon Sep 17 00:00:00 2001 From: Micah Fitch Date: Wed, 25 Jul 2018 16:06:44 -0500 Subject: [PATCH 08/28] use `==` instead of `=` in template `if` statement --- .../contentcuration/templates/settings/settings.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/contentcuration/contentcuration/templates/settings/settings.html b/contentcuration/contentcuration/templates/settings/settings.html index b00faba141..7391292b8b 100644 --- a/contentcuration/contentcuration/templates/settings/settings.html +++ b/contentcuration/contentcuration/templates/settings/settings.html @@ -13,12 +13,12 @@
    {% trans "My Settings" %}
    From 5de82a123772a3b593bc0cd265b3a8e9802d4b94 Mon Sep 17 00:00:00 2001 From: kollivier Date: Tue, 7 Aug 2018 12:10:26 -0700 Subject: [PATCH 09/28] Re-delete requirements.txt as a result of rebasing issues. --- requirements.txt | 48 ------------------------------------------------ 1 file changed, 48 deletions(-) delete mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index be65b8840b..0000000000 --- a/requirements.txt +++ /dev/null @@ -1,48 +0,0 @@ -Django==1.11 -django-filter==0.9.2 -django-mptt==0.9.0 -django-pg-utils==0.1.5 -djangorestframework==3.8.2 -pdfkit==0.6.1 -psycopg2-binary==2.7.4 -Markdown==2.6.2 -Pillow==5.0.0 -psutil==3.1.1 -django-js-reverse==0.6.1 -django-registration==2.1.2 -django-hashedfilenamestorage==1.0.0 -djangorestframework-bulk==0.2.1 -django-email-extras==0.3.3 -kolibri==0.7.0 -validators -le-utils>=0.1.11 -gunicorn==19.6.0 -django-mailgun==0.9.1 -pressurecooker==0.0.15 -metaphone==0.6 -porter2stemmer==1.0 -django-postmark==0.1.6 -ddtrace==0.8.0 -jsonfield==2.0.2 -newrelic==2.86.3.70 -celery==4.1.1 -redis==2.10.5 -reportlab==3.4.0 -python-resize-image==1.1.11 -pycountry==17.5.14 -sphinx==1.6.4 -sphinx-autobuild==0.7.1 -sphinx_rtd_theme -sphinx-intl -django-s3-storage==0.12.4 -boto3==1.6.5 -requests==2.19.1 -minio==3.0.3 -pathlib -sqlalchemy==1.2.2 -xhtml2pdf==0.2.1 -progressbar2==3.38.0 -python-postmark==0.5.0 -django-webpack-loader==0.6.0 -gspread==3.0.1 -google-api-python-client==1.6.4 From 030fb4115f324f31a24f0bd4c26a15cf58d6626b Mon Sep 17 00:00:00 2001 From: kollivier Date: Tue, 7 Aug 2018 13:02:52 -0700 Subject: [PATCH 10/28] Update Django to 1.11 in pipfile after rebase. --- Pipfile | 2 +- Pipfile.lock | 62 ++++++++++++++++++++++++++++------------------------ k8s/Makefile | 11 +++++----- 3 files changed, 41 insertions(+), 34 deletions(-) diff --git a/Pipfile b/Pipfile index 448f12bc83..5c77e068e4 100644 --- a/Pipfile +++ b/Pipfile @@ -39,7 +39,7 @@ pathlib = "*" "xhtml2pdf" = "==0.2.1" "progressbar2" = "==3.36.0" python-postmark = "==0.5.0" -Django = "==1.9.13" +Django = "==1.11" Markdown = "==2.6.2" Pillow = "==5.0.0" Metaphone = "==0.6" diff --git a/Pipfile.lock b/Pipfile.lock index 0f8d48b67c..7feb8fc3c3 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "ee7b54896da973cf82317e3f65dbb49442e89f2a3139385cff3553c4854c0580" + "sha256": "4b1a9637636385b747eb138ddc14c043562e8afb937a8e8d90a89a69d4a68335" }, "pipfile-spec": 6, "requires": { @@ -63,6 +63,7 @@ "sha256:7b56b5ca3a3917be7af26bdbf6d7da99f7319652dfe60774a4ae14fb1eaeb5e6" ], "version": "==1.12.3" + }, "cachetools": { "hashes": [ @@ -131,11 +132,11 @@ }, "django": { "hashes": [ - "sha256:c007dba5086061f7d0f4d88a3bc4016d881a7eede86d6c1c4fdbbaadddd53f1d", - "sha256:c213590aa8599b17a9a914ab017f7dc6fc16c9c69603ecf071100346b8d9d777" + "sha256:0120b3b60760fb0617848b58aaa9702c0bf963320ed472f0879c5c55ab75b64a", + "sha256:b6f3b864944276b4fd1d099952112696558f78b77b39188ac92b6c5e80152c30" ], "index": "pypi", - "version": "==1.9.13" + "version": "==1.11" }, "django-db-readonly": { "hashes": [ @@ -351,6 +352,7 @@ "hashes": [ "sha256:c075eddaa2628ab519e01b7d75b76e66c40eaa50fc52758d8225f84708950ef2" ], + "markers": "python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*' and python_version != '3.3.*'", "version": "==1.5.3" }, "grpcio": { @@ -483,6 +485,7 @@ "sha256:583179dc8d49b040a9da79bd33de59e160d2a8802b939e304eb359a4419f6498", "sha256:dd4469a8f5a6833576e9f5433f1439c306de15dbbfeceabd32479b1123380fa5" ], + "markers": "python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*' and python_version != '3.3.*'", "version": "==2.5.2" }, "markdown": { @@ -650,6 +653,7 @@ "sha256:ddd16ab250b4fc97db1c47407e78c25216a75c29d29d10ad37e51b7a2ec7b2c3" ], "index": "pypi", + "markers": "python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*' and python_version != '3.3.*'", "version": "==5.0.0" }, "port-for": { @@ -667,10 +671,11 @@ }, "pressurecooker": { "hashes": [ - "sha256:cc474f999a133ac974f33ed59d6b0be773a5c4b12e6aed44e0231d1f810ea42a" + "sha256:2ccdc7fee4a32ff9bfd980578dd005f2cacf39ead0a7dd439e00fbfc8b2edcf9", + "sha256:63a7fa5ece66617140388971772590aa8598809a3e4397887eaa2bde7661e372" ], "index": "pypi", - "version": "==0.0.18" + "version": "==0.0.15" }, "progressbar2": { "hashes": [ @@ -682,23 +687,23 @@ }, "protobuf": { "hashes": [ - "sha256:10394a4d03af7060fa8a6e1cbf38cea44be1467053b0aea5bbfcb4b13c4b88c4", - "sha256:1489b376b0f364bcc6f89519718c057eb191d7ad6f1b395ffd93d1aa45587811", - "sha256:1931d8efce896981fe410c802fd66df14f9f429c32a72dd9cfeeac9815ec6444", - "sha256:196d3a80f93c537f27d2a19a4fafb826fb4c331b0b99110f985119391d170f96", - "sha256:46e34fdcc2b1f2620172d3a4885128705a4e658b9b62355ae5e98f9ea19f42c2", - "sha256:59cd75ded98094d3cf2d79e84cdb38a46e33e7441b2826f3838dcc7c07f82995", - "sha256:5ee0522eed6680bb5bac5b6d738f7b0923b3cafce8c4b1a039a6107f0841d7ed", - "sha256:65917cfd5da9dfc993d5684643063318a2e875f798047911a9dd71ca066641c9", - "sha256:685bc4ec61a50f7360c9fd18e277b65db90105adbf9c79938bd315435e526b90", - "sha256:92e8418976e52201364a3174e40dc31f5fd8c147186d72380cbda54e0464ee19", - "sha256:9335f79d1940dfb9bcaf8ec881fb8ab47d7a2c721fb8b02949aab8bbf8b68625", - "sha256:a7ee3bb6de78185e5411487bef8bc1c59ebd97e47713cba3c460ef44e99b3db9", - "sha256:ceec283da2323e2431c49de58f80e1718986b79be59c266bb0509cbf90ca5b9e", - "sha256:fcfc907746ec22716f05ea96b7f41597dfe1a1c088f861efb8a0d4f4196a6f10" + "sha256:12985d9f40c104da2f44ec089449214876809b40fdc5d9e43b93b512b9e74056", + "sha256:12c97fe27af12fc5d66b23f905ab09dd4fb0c68d5a74a419d914580e6d2e71e3", + "sha256:327fb9d8a8247bc780b9ea7ed03c0643bc0d22c139b761c9ec1efc7cc3f0923e", + "sha256:3895319db04c0b3baed74fb66be7ba9f4cd8e88a432b8e71032cdf08b2dfee23", + "sha256:695072063e256d32335d48b9484451f7c7948edc3dbd419469d6a778602682fc", + "sha256:7d786f3ef5b33a04e6538089674f244a3b0f588155016559d950989010af97d0", + "sha256:8bf82bb7a466a54be7272dcb492f71d55a2453a58d862fb74c3f2083f2768543", + "sha256:9bbc1ae1c33c1bd3a2fc05a3aec328544d2b039ff0ce6f000063628a32fad777", + "sha256:9f1087abb67b34e55108bc610936b34363a7aac692023bcbb17e065c253a1f80", + "sha256:9fefcb92a3784b446abf3641d9a14dad815bee88e0edd10b9a9e0e144d01a991", + "sha256:a37836aa47d1b81c2db1a6b7a5e79926062b5d76bd962115a0e615551be2b48d", + "sha256:cca22955443c55cf86f963a4ad7057bca95e4dcde84d6a493066d380cfab3bb0", + "sha256:d7ac50bc06d31deb07ace6de85556c1d7330e5c0958f3b2af85037d6d1182abf", + "sha256:dfe6899304b898538f4dc94fa0b281b56b70e40f58afa4c6f807805261cbe2e8" ], "markers": "python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*' and python_version != '3.3.*'", - "version": "==3.6.1" + "version": "==3.6.0" }, "psutil": { "hashes": [ @@ -971,6 +976,7 @@ "sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd", "sha256:9de47f375baf1ea07cdb3436ff39d7a9c76042c10a769c52353ec46e4e8fc3b9" ], + "markers": "python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*' and python_version != '3.3.*'", "version": "==1.1.0" }, "sqlalchemy": { @@ -985,6 +991,7 @@ "sha256:24b66882f5f7aaedcd46b4669322e88d639d46c6ce4679ae7f397bbe4fe9a529", "sha256:eb2b989cf03ffc7166339eb34f1aa26c5ace255243337b1e22dab7caa1166687" ], + "markers": "python_version != '3.2.*' and python_version != '3.0.*' and python_version != '3.1.*' and python_version < '4' and python_version >= '2.6'", "version": "==3.5.2" }, "tornado": { @@ -1140,11 +1147,11 @@ }, "django": { "hashes": [ - "sha256:c007dba5086061f7d0f4d88a3bc4016d881a7eede86d6c1c4fdbbaadddd53f1d", - "sha256:c213590aa8599b17a9a914ab017f7dc6fc16c9c69603ecf071100346b8d9d777" + "sha256:0120b3b60760fb0617848b58aaa9702c0bf963320ed472f0879c5c55ab75b64a", + "sha256:b6f3b864944276b4fd1d099952112696558f78b77b39188ac92b6c5e80152c30" ], "index": "pypi", - "version": "==1.9.13" + "version": "==1.11" }, "django-debug-panel": { "hashes": [ @@ -1197,7 +1204,7 @@ "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" ], - "markers": "python_version < '3.0'", + "markers": "python_version < '3.3'", "version": "==1.0.2" }, "future": { @@ -1352,7 +1359,6 @@ "sha256:6e3836e39f4d36ae72840833db137f7b7d35105079aee6ec4a62d9f80d594dd1", "sha256:95eb8364a4708392bae89035f45341871286a333f749c3141c20573d2b3876e1" ], - "markers": "python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*' and python_version != '3.3.*'", "version": "==0.7.1" }, "py": { @@ -1477,10 +1483,10 @@ }, "rope": { "hashes": [ - "sha256:a108c445e1cd897fe19272ab7877d172e7faf3d4148c80e7d20faba42ea8f7b2" + "sha256:a09edfd2034fd50099a67822f9bd851fbd0f4e98d3b87519f6267b60e50d80d1" ], "index": "pypi", - "version": "==0.11.0" + "version": "==0.10.7" }, "scandir": { "hashes": [ diff --git a/k8s/Makefile b/k8s/Makefile index 53066de675..c81ba867d9 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -1,6 +1,7 @@ -build: - make -C images/app/ imagebuild - make -C images/nginx/ imagebuild +DEPLOYMENT := `kubectl get deploy -l app=master-studio -o custom-columns=NAME:.metadata.name --no-headers` +POD := `kubectl get pods -o=custom-columns=NAME:.metadata.name --field-selector=status.phase=Running --no-headers -l app=master-studio | head -n 1` -setup: - minikube addons enable freshpod +master-shell: + kubectl rollout status deployment/$(DEPLOYMENT) + echo Running bash inside $(POD) + kubectl exec -it $(POD) bash From a05dc76831528a89b47427004b6870859b1b553f Mon Sep 17 00:00:00 2001 From: kollivier Date: Tue, 7 Aug 2018 13:35:39 -0700 Subject: [PATCH 11/28] Actually update the dependencies after accidentally using pip... --- Pipfile | 4 ++-- Pipfile.lock | 27 +++++++++++++++++---------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/Pipfile b/Pipfile index 5c77e068e4..96fce417c7 100644 --- a/Pipfile +++ b/Pipfile @@ -5,9 +5,9 @@ name = "pypi" [packages] django-filter = "==0.9.2" -django-mptt = "==0.8.5" +django-mptt = "==0.9.1" django-pg-utils = "==0.1.5" -djangorestframework = "==3.3.3" +djangorestframework = "==3.8.2" pdfkit = "==0.6.1" "psycopg2-binary" = "==2.7.4" psutil = "==3.1.1" diff --git a/Pipfile.lock b/Pipfile.lock index 7feb8fc3c3..66bc1e1ce3 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "4b1a9637636385b747eb138ddc14c043562e8afb937a8e8d90a89a69d4a68335" + "sha256": "b0008db2586bde282b7589be003bb58d0a638602dfa6d7cb38bb8145be35a089" }, "pipfile-spec": 6, "requires": { @@ -169,6 +169,13 @@ "index": "pypi", "version": "==1.0.0" }, + "django-js-asset": { + "hashes": [ + "sha256:30149158206f693a5d027fe590096fc84495486bd11cd77d395b4f2ec27fc1d0", + "sha256:a395d8d19eb201ea8d2bd4f145b38f1717cd74c0f609f040141d8724c5a27f36" + ], + "version": "==1.1.0" + }, "django-js-reverse": { "hashes": [ "sha256:62c540d46b1de17adcbb80ea6ef7239b4ab9793fadae1595383b2be801b1af60", @@ -186,11 +193,11 @@ }, "django-mptt": { "hashes": [ - "sha256:829724ac56b941f0b8ceb5ec4a32c7575f92eceb86d2558a5d0a2dc76cabe45a", - "sha256:f83a47f917b27c6e46f837e7fb88b3bf8eabfed14d71edcfb21937b32a713c6b" + "sha256:18a41d1b56ca7c02a5b04d246e33ee2d18f6ee5459c02ed1d945f5abdef23a2e", + "sha256:689a04cce0981671d6061a9928c33a16b47abb0d4cd43cf7dec31ae284fdae9d" ], "index": "pypi", - "version": "==0.8.5" + "version": "==0.9.1" }, "django-pg-utils": { "hashes": [ @@ -230,11 +237,11 @@ }, "djangorestframework": { "hashes": [ - "sha256:4f47056ad798103fc9fb049dff8a67a91963bd215d31bad12ad72b891559ab16", - "sha256:f606f2bb4e9bb320937cb6ccce299991b2d302f5cc705a671dffca491e55935c" + "sha256:b6714c3e4b0f8d524f193c91ecf5f5450092c2145439ac2769711f7eba89a9d9", + "sha256:c375e4f95a3a64fccac412e36fb42ba36881e52313ec021ef410b40f67cddca4" ], "index": "pypi", - "version": "==3.3.3" + "version": "==3.8.2" }, "djangorestframework-bulk": { "hashes": [ @@ -1128,7 +1135,7 @@ "sha256:9d98697f088eb1b0fa451391f91afb5e3ebde16bbdb272819fd091151fda4f1a", "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" ], - "markers": "python_version >= '2.6'", + "markers": "python_version < '3.4'", "version": "==1.5" }, "colorama": { @@ -1142,7 +1149,7 @@ "hashes": [ "sha256:5308b47021bc2340965c371f0f058cc6971a04502638d4244225c49d80db273a" ], - "markers": "python_version < '3.2'", + "markers": "python_version == '2.7'", "version": "==3.5.0" }, "django": { @@ -1204,7 +1211,7 @@ "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" ], - "markers": "python_version < '3.3'", + "markers": "python_version < '3.0'", "version": "==1.0.2" }, "future": { From 7a07559b3f7bdd455a66ff9ce73bf60b382f0a2b Mon Sep 17 00:00:00 2001 From: Aron Fyodor Asor <191955+aronasorman@users.noreply.github.com> Date: Wed, 15 Aug 2018 17:49:38 -0700 Subject: [PATCH 12/28] update django to 1.11.15! --- Pipfile | 2 +- Pipfile.lock | 62 ++++++++++++++++++++++++---------------------------- 2 files changed, 29 insertions(+), 35 deletions(-) diff --git a/Pipfile b/Pipfile index 96fce417c7..bbe77c600d 100644 --- a/Pipfile +++ b/Pipfile @@ -39,7 +39,7 @@ pathlib = "*" "xhtml2pdf" = "==0.2.1" "progressbar2" = "==3.36.0" python-postmark = "==0.5.0" -Django = "==1.11" +Django = "==1.11.15" Markdown = "==2.6.2" Pillow = "==5.0.0" Metaphone = "==0.6" diff --git a/Pipfile.lock b/Pipfile.lock index 66bc1e1ce3..a432740e63 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "b0008db2586bde282b7589be003bb58d0a638602dfa6d7cb38bb8145be35a089" + "sha256": "fdc95fe67d9fd9124888f4804a0424d0c6fe648c0dbb185671ea0cb76622668a" }, "pipfile-spec": 6, "requires": { @@ -63,7 +63,6 @@ "sha256:7b56b5ca3a3917be7af26bdbf6d7da99f7319652dfe60774a4ae14fb1eaeb5e6" ], "version": "==1.12.3" - }, "cachetools": { "hashes": [ @@ -132,11 +131,11 @@ }, "django": { "hashes": [ - "sha256:0120b3b60760fb0617848b58aaa9702c0bf963320ed472f0879c5c55ab75b64a", - "sha256:b6f3b864944276b4fd1d099952112696558f78b77b39188ac92b6c5e80152c30" + "sha256:8176ac7985fe6737ce3d6b2531b4a2453cb7c3377c9db00bacb2b3320f4a1311", + "sha256:b18235d82426f09733d2de9910cee975cf52ff05e5f836681eb957d105a05a40" ], "index": "pypi", - "version": "==1.11" + "version": "==1.11.15" }, "django-db-readonly": { "hashes": [ @@ -359,7 +358,6 @@ "hashes": [ "sha256:c075eddaa2628ab519e01b7d75b76e66c40eaa50fc52758d8225f84708950ef2" ], - "markers": "python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*' and python_version != '3.3.*'", "version": "==1.5.3" }, "grpcio": { @@ -492,7 +490,6 @@ "sha256:583179dc8d49b040a9da79bd33de59e160d2a8802b939e304eb359a4419f6498", "sha256:dd4469a8f5a6833576e9f5433f1439c306de15dbbfeceabd32479b1123380fa5" ], - "markers": "python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*' and python_version != '3.3.*'", "version": "==2.5.2" }, "markdown": { @@ -660,7 +657,6 @@ "sha256:ddd16ab250b4fc97db1c47407e78c25216a75c29d29d10ad37e51b7a2ec7b2c3" ], "index": "pypi", - "markers": "python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*' and python_version != '3.3.*'", "version": "==5.0.0" }, "port-for": { @@ -678,11 +674,10 @@ }, "pressurecooker": { "hashes": [ - "sha256:2ccdc7fee4a32ff9bfd980578dd005f2cacf39ead0a7dd439e00fbfc8b2edcf9", - "sha256:63a7fa5ece66617140388971772590aa8598809a3e4397887eaa2bde7661e372" + "sha256:cc474f999a133ac974f33ed59d6b0be773a5c4b12e6aed44e0231d1f810ea42a" ], "index": "pypi", - "version": "==0.0.15" + "version": "==0.0.18" }, "progressbar2": { "hashes": [ @@ -694,23 +689,23 @@ }, "protobuf": { "hashes": [ - "sha256:12985d9f40c104da2f44ec089449214876809b40fdc5d9e43b93b512b9e74056", - "sha256:12c97fe27af12fc5d66b23f905ab09dd4fb0c68d5a74a419d914580e6d2e71e3", - "sha256:327fb9d8a8247bc780b9ea7ed03c0643bc0d22c139b761c9ec1efc7cc3f0923e", - "sha256:3895319db04c0b3baed74fb66be7ba9f4cd8e88a432b8e71032cdf08b2dfee23", - "sha256:695072063e256d32335d48b9484451f7c7948edc3dbd419469d6a778602682fc", - "sha256:7d786f3ef5b33a04e6538089674f244a3b0f588155016559d950989010af97d0", - "sha256:8bf82bb7a466a54be7272dcb492f71d55a2453a58d862fb74c3f2083f2768543", - "sha256:9bbc1ae1c33c1bd3a2fc05a3aec328544d2b039ff0ce6f000063628a32fad777", - "sha256:9f1087abb67b34e55108bc610936b34363a7aac692023bcbb17e065c253a1f80", - "sha256:9fefcb92a3784b446abf3641d9a14dad815bee88e0edd10b9a9e0e144d01a991", - "sha256:a37836aa47d1b81c2db1a6b7a5e79926062b5d76bd962115a0e615551be2b48d", - "sha256:cca22955443c55cf86f963a4ad7057bca95e4dcde84d6a493066d380cfab3bb0", - "sha256:d7ac50bc06d31deb07ace6de85556c1d7330e5c0958f3b2af85037d6d1182abf", - "sha256:dfe6899304b898538f4dc94fa0b281b56b70e40f58afa4c6f807805261cbe2e8" + "sha256:10394a4d03af7060fa8a6e1cbf38cea44be1467053b0aea5bbfcb4b13c4b88c4", + "sha256:1489b376b0f364bcc6f89519718c057eb191d7ad6f1b395ffd93d1aa45587811", + "sha256:1931d8efce896981fe410c802fd66df14f9f429c32a72dd9cfeeac9815ec6444", + "sha256:196d3a80f93c537f27d2a19a4fafb826fb4c331b0b99110f985119391d170f96", + "sha256:46e34fdcc2b1f2620172d3a4885128705a4e658b9b62355ae5e98f9ea19f42c2", + "sha256:59cd75ded98094d3cf2d79e84cdb38a46e33e7441b2826f3838dcc7c07f82995", + "sha256:5ee0522eed6680bb5bac5b6d738f7b0923b3cafce8c4b1a039a6107f0841d7ed", + "sha256:65917cfd5da9dfc993d5684643063318a2e875f798047911a9dd71ca066641c9", + "sha256:685bc4ec61a50f7360c9fd18e277b65db90105adbf9c79938bd315435e526b90", + "sha256:92e8418976e52201364a3174e40dc31f5fd8c147186d72380cbda54e0464ee19", + "sha256:9335f79d1940dfb9bcaf8ec881fb8ab47d7a2c721fb8b02949aab8bbf8b68625", + "sha256:a7ee3bb6de78185e5411487bef8bc1c59ebd97e47713cba3c460ef44e99b3db9", + "sha256:ceec283da2323e2431c49de58f80e1718986b79be59c266bb0509cbf90ca5b9e", + "sha256:fcfc907746ec22716f05ea96b7f41597dfe1a1c088f861efb8a0d4f4196a6f10" ], "markers": "python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*' and python_version != '3.3.*'", - "version": "==3.6.0" + "version": "==3.6.1" }, "psutil": { "hashes": [ @@ -983,7 +978,6 @@ "sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd", "sha256:9de47f375baf1ea07cdb3436ff39d7a9c76042c10a769c52353ec46e4e8fc3b9" ], - "markers": "python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*' and python_version != '3.3.*'", "version": "==1.1.0" }, "sqlalchemy": { @@ -998,7 +992,6 @@ "sha256:24b66882f5f7aaedcd46b4669322e88d639d46c6ce4679ae7f397bbe4fe9a529", "sha256:eb2b989cf03ffc7166339eb34f1aa26c5ace255243337b1e22dab7caa1166687" ], - "markers": "python_version != '3.2.*' and python_version != '3.0.*' and python_version != '3.1.*' and python_version < '4' and python_version >= '2.6'", "version": "==3.5.2" }, "tornado": { @@ -1149,16 +1142,16 @@ "hashes": [ "sha256:5308b47021bc2340965c371f0f058cc6971a04502638d4244225c49d80db273a" ], - "markers": "python_version == '2.7'", + "markers": "python_version < '3.2'", "version": "==3.5.0" }, "django": { "hashes": [ - "sha256:0120b3b60760fb0617848b58aaa9702c0bf963320ed472f0879c5c55ab75b64a", - "sha256:b6f3b864944276b4fd1d099952112696558f78b77b39188ac92b6c5e80152c30" + "sha256:8176ac7985fe6737ce3d6b2531b4a2453cb7c3377c9db00bacb2b3320f4a1311", + "sha256:b18235d82426f09733d2de9910cee975cf52ff05e5f836681eb957d105a05a40" ], "index": "pypi", - "version": "==1.11" + "version": "==1.11.15" }, "django-debug-panel": { "hashes": [ @@ -1366,6 +1359,7 @@ "sha256:6e3836e39f4d36ae72840833db137f7b7d35105079aee6ec4a62d9f80d594dd1", "sha256:95eb8364a4708392bae89035f45341871286a333f749c3141c20573d2b3876e1" ], + "markers": "python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*' and python_version != '3.3.*'", "version": "==0.7.1" }, "py": { @@ -1490,10 +1484,10 @@ }, "rope": { "hashes": [ - "sha256:a09edfd2034fd50099a67822f9bd851fbd0f4e98d3b87519f6267b60e50d80d1" + "sha256:a108c445e1cd897fe19272ab7877d172e7faf3d4148c80e7d20faba42ea8f7b2" ], "index": "pypi", - "version": "==0.10.7" + "version": "==0.11.0" }, "scandir": { "hashes": [ From 5119abbba3570efa42b894170530fa7b47a474ce Mon Sep 17 00:00:00 2001 From: Aron Fyodor Asor <191955+aronasorman@users.noreply.github.com> Date: Wed, 15 Aug 2018 17:49:57 -0700 Subject: [PATCH 13/28] use the recommended way of declaring our AppConfig Through our INSTALLED_APPS! See https://docs.djangoproject.com/en/1.11/ref/applications/#configuring-applications --- contentcuration/contentcuration/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/contentcuration/contentcuration/__init__.py b/contentcuration/contentcuration/__init__.py index 5924221adc..b64e43e83b 100644 --- a/contentcuration/contentcuration/__init__.py +++ b/contentcuration/contentcuration/__init__.py @@ -3,5 +3,3 @@ # This will make sure the app is always imported when # Django starts so that shared_task will use this app. from .celery import app as celery_app - -default_app_config = "contentcuration.apps.ContentConfig" From c8537ddf49b70b4aa7ca9d9e5a8b2477fa9bd12d Mon Sep 17 00:00:00 2001 From: Micah Fitch Date: Thu, 16 Aug 2018 18:16:12 -0400 Subject: [PATCH 14/28] use FormatPresetSerializer as a PrimaryKeyRelatedField serializer --- contentcuration/contentcuration/serializers.py | 6 ++++-- contentcuration/contentcuration/utils/files.py | 4 +++- contentcuration/contentcuration/views/files.py | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/contentcuration/contentcuration/serializers.py b/contentcuration/contentcuration/serializers.py index bb45b149b2..dcffdcc6cf 100644 --- a/contentcuration/contentcuration/serializers.py +++ b/contentcuration/contentcuration/serializers.py @@ -51,7 +51,7 @@ class Meta: fields = ("__all__") -class FormatPresetSerializer(serializers.ModelSerializer): +class FormatPresetSerializer(serializers.PrimaryKeyRelatedField): # files = FileSerializer(many=True, read_only=True) associated_mimetypes = serializers.SerializerMethodField('retrieve_mimetypes') # Handles multi-language content (Backbone won't allow duplicate ids in collection, so name retains id) @@ -77,6 +77,7 @@ def update(self, instance, validated_data): user = self.context['request'].user with transaction.atomic(): for item in validated_data: + # import ipdb; ipdb.set_trace() item.update({ 'preset_id': item['preset']['id'], 'language_id': item.get('language')['id'] if item.get('language') else None @@ -149,7 +150,8 @@ class FileSerializer(BulkSerializerMixin, serializers.ModelSerializer): language = LanguageSerializer(many=False, required=False, allow_null=True) display_name = serializers.SerializerMethodField('retrieve_display_name') id = serializers.CharField(required=False) - preset = FormatPresetSerializer(many=False) + preset = FormatPresetSerializer(many=False, read_only=True) + # preset = serializers.PrimaryKeyRelatedField(many=False, queryset=FormatPreset.objects.all()) def get(*args, **kwargs): return super.get(*args, **kwargs) diff --git a/contentcuration/contentcuration/utils/files.py b/contentcuration/contentcuration/utils/files.py index be0df24140..10d8379994 100644 --- a/contentcuration/contentcuration/utils/files.py +++ b/contentcuration/contentcuration/utils/files.py @@ -23,7 +23,8 @@ def create_file_from_contents(contents, ext=None, node=None, preset_id=None, uploaded_by=None): checksum, _, path = write_raw_content_to_storage(contents, ext=ext) with default_storage.open(path, 'rb') as new_file: - return File.objects.create( + import ipdb; ipdb.set_trace() + result = File.objects.create( file_on_disk=DjFile(new_file), file_format_id=ext, file_size=default_storage.size(path), @@ -32,6 +33,7 @@ def create_file_from_contents(contents, ext=None, node=None, preset_id=None, upl contentnode=node, uploaded_by=uploaded_by ) + return result def get_file_diff(files): diff --git a/contentcuration/contentcuration/views/files.py b/contentcuration/contentcuration/views/files.py index 90ae903ea9..5bfc2e8aab 100644 --- a/contentcuration/contentcuration/views/files.py +++ b/contentcuration/contentcuration/views/files.py @@ -55,6 +55,7 @@ def file_create(request): if request.method != 'POST': return HttpResponseBadRequest("Only POST requests are allowed on this endpoint.") + import ipdb; ipdb.set_trace() original_filename, ext = os.path.splitext(request.FILES.values()[0]._name) size = request.FILES.values()[0]._size contentfile = DjFile(request.FILES.values()[0]) From 3df6e23f76f1c76f7d71befb73101bca8e8da2a7 Mon Sep 17 00:00:00 2001 From: Micah Fitch Date: Thu, 16 Aug 2018 18:40:20 -0400 Subject: [PATCH 15/28] disable breakpoints --- contentcuration/contentcuration/utils/files.py | 2 +- contentcuration/contentcuration/views/files.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contentcuration/contentcuration/utils/files.py b/contentcuration/contentcuration/utils/files.py index 10d8379994..86ed2df200 100644 --- a/contentcuration/contentcuration/utils/files.py +++ b/contentcuration/contentcuration/utils/files.py @@ -23,7 +23,7 @@ def create_file_from_contents(contents, ext=None, node=None, preset_id=None, uploaded_by=None): checksum, _, path = write_raw_content_to_storage(contents, ext=ext) with default_storage.open(path, 'rb') as new_file: - import ipdb; ipdb.set_trace() + # import ipdb; ipdb.set_trace() result = File.objects.create( file_on_disk=DjFile(new_file), file_format_id=ext, diff --git a/contentcuration/contentcuration/views/files.py b/contentcuration/contentcuration/views/files.py index 5bfc2e8aab..554f7b3f9e 100644 --- a/contentcuration/contentcuration/views/files.py +++ b/contentcuration/contentcuration/views/files.py @@ -55,7 +55,7 @@ def file_create(request): if request.method != 'POST': return HttpResponseBadRequest("Only POST requests are allowed on this endpoint.") - import ipdb; ipdb.set_trace() + # import ipdb; ipdb.set_trace() original_filename, ext = os.path.splitext(request.FILES.values()[0]._name) size = request.FILES.values()[0]._size contentfile = DjFile(request.FILES.values()[0]) From f3054084bfc0bc02a724f33aa9c4a27f367664cd Mon Sep 17 00:00:00 2001 From: Micah Fitch Date: Tue, 28 Aug 2018 21:45:34 -0400 Subject: [PATCH 16/28] attempted fix for file uploading issues --- .../contentcuration/serializers.py | 79 ++++++++++--------- .../static/js/bundle_modules/base.js | 2 +- .../static/js/edit_channel/models.js | 5 +- 3 files changed, 46 insertions(+), 40 deletions(-) diff --git a/contentcuration/contentcuration/serializers.py b/contentcuration/contentcuration/serializers.py index dcffdcc6cf..b9de18abb1 100644 --- a/contentcuration/contentcuration/serializers.py +++ b/contentcuration/contentcuration/serializers.py @@ -51,7 +51,7 @@ class Meta: fields = ("__all__") -class FormatPresetSerializer(serializers.PrimaryKeyRelatedField): +class FormatPresetSerializer(serializers.ModelSerializer): # files = FileSerializer(many=True, read_only=True) associated_mimetypes = serializers.SerializerMethodField('retrieve_mimetypes') # Handles multi-language content (Backbone won't allow duplicate ids in collection, so name retains id) @@ -75,23 +75,24 @@ def update(self, instance, validated_data): ret = [] update_files = {} user = self.context['request'].user + import ipdb with transaction.atomic(): for item in validated_data: - # import ipdb; ipdb.set_trace() - item.update({ - 'preset_id': item['preset']['id'], - 'language_id': item.get('language')['id'] if item.get('language') else None - }) - - # User should not be able to change files without a display - if item['preset']['display']: - if 'id' in item: - update_files[item['id']] = item - else: - # create new nodes - ret.append(File.objects.create(**item)) - item.pop('preset', None) - item.pop('language', None) + ipdb.set_trace() + # item.update({ + # # 'preset_id': item['preset']['id'], + # 'language_id': item.get('language')['id'] if item.get('language') else None + # }) + + # # User should not be able to change files without a display + # if item['preset']['display']: + # if 'id' in item: + # update_files[item['id']] = item + # else: + # # create new nodes + # ret.append(File.objects.create(**item)) + # item.pop('preset', None) + # item.pop('language', None) files_to_delete = [] nodes_to_parse = [] @@ -99,26 +100,30 @@ def update(self, instance, validated_data): # Get files that have the same contentnode, preset, and language as the files that are now attached to this node for file_obj in validated_data: - delete_queryset = File.objects.filter( - Q(contentnode=file_obj['contentnode']) & # Get files that are associated with this node - (Q(preset_id=file_obj['preset_id']) | Q( - preset=None)) & # Look at files that have the same preset as this file - Q(language_id=file_obj.get('language_id')) & # Look at files with the same language as this file - ~Q(id=file_obj['id']) # Remove the file if it's not this file - ) - files_to_delete += [f for f in delete_queryset.all()] - if file_obj['contentnode'] not in nodes_to_parse: - nodes_to_parse.append(file_obj['contentnode']) - - # Delete removed files - for node in nodes_to_parse: - previous_files = node.files.all() - for f in previous_files: - if f.id not in current_files: - files_to_delete.append(f) - - for to_delete in files_to_delete: - to_delete.delete() + file_obj = File.objects.get(pk=file_obj['id']) + file_obj = file_obj.contentnode.files.exclude(pk=file_obj.pk)\ + .filter(preset_id=file_obj.preset_id, language_id=file_obj.language_id)\ + .delete() + # delete_queryset = File.objects.filter( + # Q(contentnode=file_obj['contentnode']) & # Get files that are associated with this node + # (Q(preset_id=file_obj['preset_id']) | Q( + # preset=None)) & # Look at files that have the same preset as this file + # Q(language_id=file_obj.get('language_id')) & # Look at files with the same language as this file + # ~Q(id=file_obj['id']) # Remove the file if it's not this file + # ) + # files_to_delete += [f for f in delete_queryset.all()] + # if file_obj['contentnode'] not in nodes_to_parse: + # nodes_to_parse.append(file_obj['contentnode']) + + # # Delete removed files + # for node in nodes_to_parse: + # previous_files = node.files.all() + # for f in previous_files: + # if f.id not in current_files: + # files_to_delete.append(f) + + # for to_delete in files_to_delete: + # to_delete.delete() if update_files: with transaction.atomic(): @@ -151,7 +156,7 @@ class FileSerializer(BulkSerializerMixin, serializers.ModelSerializer): display_name = serializers.SerializerMethodField('retrieve_display_name') id = serializers.CharField(required=False) preset = FormatPresetSerializer(many=False, read_only=True) - # preset = serializers.PrimaryKeyRelatedField(many=False, queryset=FormatPreset.objects.all()) + # preset_id = serializers.PrimaryKeyRelatedField(many=False, queryset=FormatPreset.objects.all()) def get(*args, **kwargs): return super.get(*args, **kwargs) diff --git a/contentcuration/contentcuration/static/js/bundle_modules/base.js b/contentcuration/contentcuration/static/js/bundle_modules/base.js index 4cf6e3b553..1a9343ebed 100644 --- a/contentcuration/contentcuration/static/js/bundle_modules/base.js +++ b/contentcuration/contentcuration/static/js/bundle_modules/base.js @@ -6,7 +6,7 @@ require('../handlebars/helpers.js'); // side effect: adds shared styles to the DOM require('../../less/styles.less'); -require('../utils/offline_helper'); +// require('../utils/offline_helper'); // Promise polyfill if(!global.Promise) { diff --git a/contentcuration/contentcuration/static/js/edit_channel/models.js b/contentcuration/contentcuration/static/js/edit_channel/models.js index f8d0424394..71ea64ef77 100644 --- a/contentcuration/contentcuration/static/js/edit_channel/models.js +++ b/contentcuration/contentcuration/static/js/edit_channel/models.js @@ -415,8 +415,9 @@ var ContentNodeCollection = BaseCollection.extend({ var to_add = new FileModel(file); var preset_data = to_add.get("preset"); preset_data.id = file.preset.name || file.preset.id; - to_add.set('preset', preset_data); - to_add.set('contentnode', node.id); + // to_add.set('preset', preset_data); + // to_add.set('preset_id', preset_data['id']); + // to_add.set('contentnode', node.id); fileCollection.add(to_add); }); assessmentCollection.add(node.get('assessment_items')); From 759da9bac35ab0844d06d84f7ccfb761ea1e0e6c Mon Sep 17 00:00:00 2001 From: Micah Date: Fri, 7 Sep 2018 23:18:44 -0400 Subject: [PATCH 17/28] [django1.11] fix file uploading, new user registration + make tests pass (#940) * Add chef method unit tests and ensure that Django views return a HTTP response when they raise errors. * Add more file upload and add node tests. * Add finish_commit tests and return a HTTP 500 code instead of raising if an error occurs. * Get Sentry reports for handled internal server errors in Internal API calls. * Add yarn run devsetup command (#925) * add new yarn command to set up dev environment * change commands to remove redundant pieces to prioritize yarn run devsetup Streamlines the new dev experience * Updated translations with epub string * Fixed default preview styling * Add caching to the get_user_public_channels endpoint (#939) * Move view get_public_channels logic to Channel model * add the Channel.make_public function * use StudioTestCase to create buckets * add ChannelCacher class to implement channel caching * add channel token related convenience functions * make exporchannel use Channel.make_token() * add channel specific cache for tokens Cache that, one less query to make. * add caching to channel token serializer endpoint * move serializers.get_resource_count implementation to the Channel model * use channel.get_thumbnail() function on serializers.py * remove redefinition of generate_thumbnail_url * add ChannelCacher.get_resource_count() cache function Used to cache the channel's get_resource_count * add caching to get_resource_count API attribute * add channel.get_date_modified function * make channelfieldmixin use channel.get_date_modified * add channel.get_date_modified cache * Use the channel cache for get_date_modified * remove redundant generate_thumbnail_url Already defined in ChannelFieldMixin * pass cache get_public_channels args to real function * cache the entire get_user_public_channels view * fix tests for ChannelCacher.get_public_channels() by comparing actual channel ids rather than objects * Create SecretToken.exists() convenience method * refactor make_token to definitely end after 100 attempts And use for-else loop construct * add extra non-public assert on the make_public test * add test for SecretToken.exists() * [WIP] Add nginx-level API endpoint caching (#943) * add long running caching to the get_user_public_channels endpoint * add caching to the get_user_edit_channels endpoint * add 4 hour browser caching on all static files (#945) * add caching to the get_user_edit_channels endpoint (#947) * Unlock le-utils' version * have pipenv update all updateable packages * Added comment * Made resource count required on storage requests * Merge in develop * Return 404s for any files not found in the zip, and have BadZipfile errors report the name, size, mode and download state to help us narrow down problems with opening. Also, tests. (#942) * Remove the hardcoded server IP in tests so that configuration changes don't break them. * Apply Micah's changes and also fix some new issues uncovered by tests to get the code to the same state as it was before revert. * Fix login template name. * Actually commit the updated dependency versions to requirements.txt... * use `==` instead of `=` in template `if` statement * Re-delete requirements.txt as a result of rebasing issues. * Update Django to 1.11 in pipfile after rebase. * Actually update the dependencies after accidentally using pip... * objects.create automatically calls save, so no need to call it again afterwards. * objects.create automatically calls save, so no need to call it again afterwards. * Revert removed calls to save as we are creating models directly rather than using objects.create. * Revert removed calls to save as we are creating models directly rather than using objects.create. * update django to 1.11.15! * use the recommended way of declaring our AppConfig Through our INSTALLED_APPS! See https://docs.djangoproject.com/en/1.11/ref/applications/#configuring-applications * use FormatPresetSerializer as a PrimaryKeyRelatedField serializer * disable breakpoints * attempted fix for file uploading issues * file uploading works... yay! * signup form multiselection area is working now * enable offline_helper * remove extra debugging leftovers from registration info template * fixes for several tests... remember: don't manually hardcode constants from le_utils. they may change some day! * fixed authentication tests * add a migration from django 1.11 --- Pipfile.lock | 52 +++++++++-------- contentcuration/contentcuration/decorators.py | 7 ++- contentcuration/contentcuration/forms.py | 1 - .../migrations/0093_auto_20180831_0745.py | 21 +++++++ contentcuration/contentcuration/models.py | 2 +- .../contentcuration/serializers.py | 57 ++++--------------- .../registration_information_form.html | 6 +- .../tests/test_authentication.py | 1 - .../tests/test_contentnodes.py | 5 ++ .../tests/test_garbage_collect.py | 14 +++-- .../tests/test_garbage_collection.py | 8 +-- .../tests/test_urlendpoints.py | 3 - .../contentcuration/tests/testdata.py | 1 + .../contentcuration/utils/files.py | 1 - 14 files changed, 87 insertions(+), 92 deletions(-) create mode 100644 contentcuration/contentcuration/migrations/0093_auto_20180831_0745.py diff --git a/Pipfile.lock b/Pipfile.lock index a432740e63..e6406c0417 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "fdc95fe67d9fd9124888f4804a0424d0c6fe648c0dbb185671ea0cb76622668a" + "sha256": "762b2fef3f9c67fe0996b5fd3ff084032a731a3799b379e7461e612103f077e4" }, "pipfile-spec": 6, "requires": { @@ -52,17 +52,17 @@ }, "boto3": { "hashes": [ - "sha256:45a24345e3f65c9a472b5e5058e7c882afe556406f0b516e4668fef3231319cf", - "sha256:f09cd3ca8f3906452e1442063ab504ea80eab4757b19dfeeb127a11b6639fe75" + "sha256:8d3bdfd7405b32040b2323522404f8198c9b5f88edd781a568f995d10c656922", + "sha256:9dd7aec0e5abbf16a64d2bf1a2f346aac0534c7d145737608d534de7a57ca8f6" ], - "version": "==1.9.3" + "version": "==1.9.5" }, "botocore": { "hashes": [ - "sha256:2a96d07e19b73d858db51d42704d5de46cfc93d6b4e4ed29d60d3d7205e6e46d", - "sha256:7b56b5ca3a3917be7af26bdbf6d7da99f7319652dfe60774a4ae14fb1eaeb5e6" + "sha256:71b7b01e186731c459b2d85efc3168668f30a1a6998a169526dbb12c321660ce", + "sha256:f2282ef08f355c8b81ba539375e98ab5345fa2569644f1aeb361ee4c6cc60bc4" ], - "version": "==1.12.3" + "version": "==1.12.5" }, "cachetools": { "hashes": [ @@ -657,6 +657,7 @@ "sha256:ddd16ab250b4fc97db1c47407e78c25216a75c29d29d10ad37e51b7a2ec7b2c3" ], "index": "pypi", + "markers": "python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*' and python_version != '3.3.*'", "version": "==5.0.0" }, "port-for": { @@ -704,7 +705,6 @@ "sha256:ceec283da2323e2431c49de58f80e1718986b79be59c266bb0509cbf90ca5b9e", "sha256:fcfc907746ec22716f05ea96b7f41597dfe1a1c088f861efb8a0d4f4196a6f10" ], - "markers": "python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*' and python_version != '3.3.*'", "version": "==3.6.1" }, "psutil": { @@ -909,10 +909,10 @@ }, "rsa": { "hashes": [ - "sha256:25df4e10c263fb88b5ace923dd84bf9aa7f5019687b5e55382ffcdb8bede9db5", - "sha256:43f682fea81c452c98d09fc316aae12de6d30c4b5c84226642cf8f8fd1c93abd" + "sha256:14ba45700ff1ec9eeb206a2ce76b32814958a98e372006c8fb76ba820211be66", + "sha256:1a836406405730121ae9823e19c6e806c62bbad73f890574fff50efa4122c487" ], - "version": "==3.4.2" + "version": "==4.0" }, "s3transfer": { "hashes": [ @@ -978,6 +978,7 @@ "sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd", "sha256:9de47f375baf1ea07cdb3436ff39d7a9c76042c10a769c52353ec46e4e8fc3b9" ], + "markers": "python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*' and python_version != '3.3.*'", "version": "==1.1.0" }, "sqlalchemy": { @@ -992,20 +993,21 @@ "sha256:24b66882f5f7aaedcd46b4669322e88d639d46c6ce4679ae7f397bbe4fe9a529", "sha256:eb2b989cf03ffc7166339eb34f1aa26c5ace255243337b1e22dab7caa1166687" ], + "markers": "python_version != '3.2.*' and python_version != '3.0.*' and python_version != '3.1.*' and python_version < '4' and python_version >= '2.6'", "version": "==3.5.2" }, "tornado": { "hashes": [ - "sha256:1c0816fc32b7d31b98781bd8ebc7a9726d7dce67407dc353a2e66e697e138448", - "sha256:4f66a2172cb947387193ca4c2c3e19131f1c70fa8be470ddbbd9317fd0801582", - "sha256:5327ba1a6c694e0149e7d9126426b3704b1d9d520852a3e4aa9fc8fe989e4046", - "sha256:6a7e8657618268bb007646b9eae7661d0b57f13efc94faa33cd2588eae5912c9", - "sha256:a9b14804783a1d77c0bd6c66f7a9b1196cbddfbdf8bceb64683c5ae60bd1ec6f", - "sha256:c58757e37c4a3172949c99099d4d5106e4d7b63aa0617f9bb24bfbff712c7866", - "sha256:d8984742ce86c0855cccecd5c6f54a9f7532c983947cff06f3a0e2115b47f85c" + "sha256:0662d28b1ca9f67108c7e3b77afabfb9c7e87bde174fbda78186ecedc2499a9d", + "sha256:4e5158d97583502a7e2739951553cbd88a72076f152b4b11b64b9a10c4c49409", + "sha256:732e836008c708de2e89a31cb2fa6c0e5a70cb60492bee6f1ea1047500feaf7f", + "sha256:8154ec22c450df4e06b35f131adc4f2f3a12ec85981a203301d310abf580500f", + "sha256:8e9d728c4579682e837c92fdd98036bd5cdefa1da2aaf6acf26947e6dd0c01c5", + "sha256:d4b3e5329f572f055b587efc57d29bd051589fb5a43ec8898c77a47ec2fa2bbb", + "sha256:e5f2585afccbff22390cddac29849df463b252b711aa2ce7c5f3f342a5b3b444" ], "markers": "python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*' and python_version != '3.3.*'", - "version": "==5.1" + "version": "==5.1.1" }, "typing": { "hashes": [ @@ -1029,6 +1031,7 @@ "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" ], + "markers": "python_version != '3.2.*' and python_version != '3.3.*' and python_version != '3.0.*' and python_version != '3.1.*' and python_version < '4' and python_version >= '2.6'", "version": "==1.23" }, "validators": { @@ -1128,7 +1131,7 @@ "sha256:9d98697f088eb1b0fa451391f91afb5e3ebde16bbdb272819fd091151fda4f1a", "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" ], - "markers": "python_version < '3.4'", + "markers": "python_version == '2.7'", "version": "==1.5" }, "colorama": { @@ -1204,7 +1207,7 @@ "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" ], - "markers": "python_version < '3.0'", + "markers": "python_version < '3.3'", "version": "==1.0.2" }, "future": { @@ -1359,7 +1362,6 @@ "sha256:6e3836e39f4d36ae72840833db137f7b7d35105079aee6ec4a62d9f80d594dd1", "sha256:95eb8364a4708392bae89035f45341871286a333f749c3141c20573d2b3876e1" ], - "markers": "python_version >= '2.7' and python_version != '3.0.*' and python_version != '3.1.*' and python_version != '3.2.*' and python_version != '3.3.*'", "version": "==0.7.1" }, "py": { @@ -1417,11 +1419,11 @@ }, "pytest-django": { "hashes": [ - "sha256:2d2e0a618d91c280d463e90bcbea9b4e417609157f611a79685b1c561c4c0836", - "sha256:59683def396923b78d7e191a7086a48193f8d5db869ace79acb38f906522bc7b" + "sha256:49e9ffc856bc6a1bec1c26c5c7b7213dff7cc8bc6b64d624c4d143d04aff0bcf", + "sha256:b379282feaf89069cb790775ab6bbbd2bd2038a68c7ef9b84a41898e0b551081" ], "index": "pypi", - "version": "==3.4.2" + "version": "==3.4.3" }, "pytest-logging": { "hashes": [ diff --git a/contentcuration/contentcuration/decorators.py b/contentcuration/contentcuration/decorators.py index e046c8991b..359de16f27 100644 --- a/contentcuration/contentcuration/decorators.py +++ b/contentcuration/contentcuration/decorators.py @@ -6,6 +6,7 @@ from django.template import RequestContext from contentcuration.models import Channel from contentcuration.utils.policies import check_policies +from django.shortcuts import render ACCEPTED_BROWSERS = settings.HEALTH_CHECK_BROWSERS + settings.SUPPORTED_BROWSERS @@ -30,7 +31,7 @@ def wrap(request, *args, **kwargs): if request.user.is_admin: return function(request, *args, **kwargs) - return render_to_response('unauthorized.html', context_instance=RequestContext(request), status=403) + return render(request, 'unauthorized.html', status=403) wrap.__doc__ = function.__doc__ wrap.__name__ = function.__name__ @@ -49,7 +50,7 @@ def wrap(request, *args, **kwargs): request.user.is_admin: return function(request, *args, **kwargs) - return render_to_response('unauthorized.html', context_instance=RequestContext(request), status=403) + return render(request, 'unauthorized.html', status=403) wrap.__doc__ = function.__doc__ wrap.__name__ = function.__name__ @@ -61,7 +62,7 @@ def wrap(request, *args, **kwargs): channel = Channel.objects.get(pk=kwargs['channel_id'], deleted=False) if not channel.editors.filter(id=request.user.id).exists() and not request.user.is_admin: - return render_to_response('unauthorized.html', context_instance=RequestContext(request), status=403) + return render(request, 'unauthorized.html', status=403) return function(request, *args, **kwargs) except ObjectDoesNotExist: diff --git a/contentcuration/contentcuration/forms.py b/contentcuration/contentcuration/forms.py index eea52f33c3..1452614e41 100644 --- a/contentcuration/contentcuration/forms.py +++ b/contentcuration/contentcuration/forms.py @@ -113,7 +113,6 @@ def __init__(self, *args, **kwargs): ) countries = [(c.name, translator.gettext(c.name)) for c in list(pycountry.countries)] - self.fields['location'] = forms.ChoiceField(required=True, widget=forms.SelectMultiple, label=_('Where do you plan to use Kolibri? (select all that apply)'), choices=countries) def clean_email(self): diff --git a/contentcuration/contentcuration/migrations/0093_auto_20180831_0745.py b/contentcuration/contentcuration/migrations/0093_auto_20180831_0745.py new file mode 100644 index 0000000000..33520ff98f --- /dev/null +++ b/contentcuration/contentcuration/migrations/0093_auto_20180831_0745.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.15 on 2018-08-31 07:45 +from __future__ import unicode_literals + +import contentcuration.models +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('contentcuration', '0092_auto_20180731_1024'), + ] + + operations = [ + migrations.AlterField( + model_name='file', + name='file_on_disk', + field=models.FileField(blank=True, max_length=500, upload_to=contentcuration.models.object_storage_name), + ), + ] diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index 70d3aaab0d..91c4b60f3a 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -232,7 +232,7 @@ def save(self, *args, **kwargs): changed = True if not self.clipboard_tree: - self.clipboard_tree = ContentNode.objects.create(title=self.email + " clipboard", kind_id="topic", + self.clipboard_tree = ContentNode.objects.create(title=self.email + " clipboard", kind_id=content_kinds.TOPIC, sort_order=get_next_sort_order()) self.clipboard_tree.save() changed = True diff --git a/contentcuration/contentcuration/serializers.py b/contentcuration/contentcuration/serializers.py index b9de18abb1..7409c4480a 100644 --- a/contentcuration/contentcuration/serializers.py +++ b/contentcuration/contentcuration/serializers.py @@ -77,53 +77,20 @@ def update(self, instance, validated_data): user = self.context['request'].user import ipdb with transaction.atomic(): + # Get files that have the same contentnode, preset, and language as the files that are now attached to this node for item in validated_data: - ipdb.set_trace() - # item.update({ - # # 'preset_id': item['preset']['id'], - # 'language_id': item.get('language')['id'] if item.get('language') else None - # }) - - # # User should not be able to change files without a display - # if item['preset']['display']: - # if 'id' in item: - # update_files[item['id']] = item - # else: - # # create new nodes - # ret.append(File.objects.create(**item)) - # item.pop('preset', None) - # item.pop('language', None) - - files_to_delete = [] - nodes_to_parse = [] - current_files = [f['id'] for f in validated_data] + file_obj = File.objects.get(pk=item['id']) + files_to_replace = item['contentnode'].files.exclude(pk=file_obj.pk)\ + .filter(preset_id=file_obj.preset_id, language_id=file_obj.language_id) + files_to_replace.delete() + + if file_obj.preset and file_obj.preset.display: + if file_obj.pk: + update_files[file_obj.pk] = item + else: + # create new nodes + ret.append(File.objects.create(**item)) - # Get files that have the same contentnode, preset, and language as the files that are now attached to this node - for file_obj in validated_data: - file_obj = File.objects.get(pk=file_obj['id']) - file_obj = file_obj.contentnode.files.exclude(pk=file_obj.pk)\ - .filter(preset_id=file_obj.preset_id, language_id=file_obj.language_id)\ - .delete() - # delete_queryset = File.objects.filter( - # Q(contentnode=file_obj['contentnode']) & # Get files that are associated with this node - # (Q(preset_id=file_obj['preset_id']) | Q( - # preset=None)) & # Look at files that have the same preset as this file - # Q(language_id=file_obj.get('language_id')) & # Look at files with the same language as this file - # ~Q(id=file_obj['id']) # Remove the file if it's not this file - # ) - # files_to_delete += [f for f in delete_queryset.all()] - # if file_obj['contentnode'] not in nodes_to_parse: - # nodes_to_parse.append(file_obj['contentnode']) - - # # Delete removed files - # for node in nodes_to_parse: - # previous_files = node.files.all() - # for f in previous_files: - # if f.id not in current_files: - # files_to_delete.append(f) - - # for to_delete in files_to_delete: - # to_delete.delete() if update_files: with transaction.atomic(): diff --git a/contentcuration/contentcuration/templates/registration/registration_information_form.html b/contentcuration/contentcuration/templates/registration/registration_information_form.html index 84deeeeeb8..cb2f26e0b5 100644 --- a/contentcuration/contentcuration/templates/registration/registration_information_form.html +++ b/contentcuration/contentcuration/templates/registration/registration_information_form.html @@ -29,11 +29,11 @@

    {% trans 'Create an Account' %}

    {% for option in form.use %}
    - +
    - - {% if option.choice_value == 'storage' %} + + {% if option.data.value == 'storage' %}
    {{form.storage.label}}*  {{form.storage}}
    diff --git a/contentcuration/contentcuration/tests/test_authentication.py b/contentcuration/contentcuration/tests/test_authentication.py index f67e09ca8e..b6d321008c 100644 --- a/contentcuration/contentcuration/tests/test_authentication.py +++ b/contentcuration/contentcuration/tests/test_authentication.py @@ -17,7 +17,6 @@ def test_authenticate_policy_update(self): if they have policies they have not yet agreed to. """ base_url = '/channels/{}'.format(self.channel.pk) - self.channel.viewers.add(self.user) self.client.force_login(self.user) diff --git a/contentcuration/contentcuration/tests/test_contentnodes.py b/contentcuration/contentcuration/tests/test_contentnodes.py index d9e5ecbaa4..ea3f7e3d67 100644 --- a/contentcuration/contentcuration/tests/test_contentnodes.py +++ b/contentcuration/contentcuration/tests/test_contentnodes.py @@ -34,6 +34,11 @@ def _check_nodes(parent, title=None, original_channel_id=None, source_channel_id class NodeOperationsTestCase(BaseTestCase): + def setUp(self): + super(NodeOperationsTestCase, self).setUp() + + self.channel = testdata.channel() + def test_duplicate_nodes(self): """ Ensures that when we copy nodes, the new channel gets marked as changed but the old channel doesn't, diff --git a/contentcuration/contentcuration/tests/test_garbage_collect.py b/contentcuration/contentcuration/tests/test_garbage_collect.py index 4b99d11f62..6860dce059 100755 --- a/contentcuration/contentcuration/tests/test_garbage_collect.py +++ b/contentcuration/contentcuration/tests/test_garbage_collect.py @@ -6,9 +6,9 @@ from django.conf import settings from django.core.files.base import ContentFile from django.core.files.storage import default_storage - from contentcuration.models import ContentNode, File from contentcuration.utils.garbage_collect import clean_up_contentnodes +from le_utils.constants import content_kinds, file_formats from base import StudioTestCase @@ -18,7 +18,7 @@ def _create_expired_contentnode(creation_date=THREE_MONTHS_AGO): c = ContentNode.objects.create( - kind_id="topic", + kind_id=content_kinds.TOPIC, title="test", modified=creation_date, created=creation_date, @@ -62,7 +62,9 @@ def test_deletes_associated_files(self): contentnode_id=c.pk, file_on_disk=ContentFile("test"), checksum="aaa", + file_format_id=file_formats.JPG ) + f.file_on_disk.save("aaa.jpg", ContentFile("aaa")) file_url = f.file_on_disk.url @@ -81,7 +83,7 @@ def test_doesnt_delete_nonorphan_files_and_contentnodes(self): # this legit tree, since it's not attached to our # orphan tree, should still exist after cleanup legit_tree = ContentNode.objects.create( - kind_id="Topic", + kind_id=content_kinds.TOPIC, ) # this file should still be here too since we attach # it to our legit tree @@ -113,7 +115,7 @@ def test_doesnt_delete_old_legit_tree(self): # our old, but not orphaned tree. This should exist at the end of our test. legit_node = ContentNode.objects.create( - kind_id="Topic", + kind_id=content_kinds.TOPIC, ) # mark the legit_node as old ContentNode.objects.filter(pk=legit_node.pk).update( @@ -141,16 +143,18 @@ def test_doesnt_delete_file_referenced_by_orphan_and_nonorphan_nodes(self): # our legit node, standing proud and high with its non-orphaned status legit_node = ContentNode.objects.create( - kind_id="Video", + kind_id=content_kinds.VIDEO, ) f = File.objects.create( contentnode=legit_node, checksum="aaa", + file_format_id=file_formats.JPG ) forphan = File.objects.create( contentnode=orphan_node, checksum="aaa", + file_format_id=file_formats.JPG ) # The file they both share. This has the same checksum and contents. diff --git a/contentcuration/contentcuration/tests/test_garbage_collection.py b/contentcuration/contentcuration/tests/test_garbage_collection.py index 97ad9db1f7..c2e78224a6 100644 --- a/contentcuration/contentcuration/tests/test_garbage_collection.py +++ b/contentcuration/contentcuration/tests/test_garbage_collection.py @@ -8,7 +8,7 @@ from contentcuration import models as cc from contentcuration.api import activate_channel from contentcuration.utils.garbage_collect import clean_up_deleted_chefs, get_deleted_chefs_root - +from le_utils.constants import content_kinds from base import BaseAPITestCase from testdata import tree @@ -22,11 +22,11 @@ class NodeSettingTestCase(BaseAPITestCase): def setUp(self): super(NodeSettingTestCase, self).setUp() # Set up ricecooker trees - self.channel.staging_tree = cc.ContentNode(kind_id="topic", title="test", node_id="aaa") + self.channel.staging_tree = cc.ContentNode(kind_id=content_kinds.TOPIC, title="test", node_id="aaa") self.channel.staging_tree.save() - self.channel.previous_tree = cc.ContentNode(kind_id="topic", title="test", node_id="bbb") + self.channel.previous_tree = cc.ContentNode(kind_id=content_kinds.TOPIC, title="test", node_id="bbb") self.channel.previous_tree.save() - self.channel.chef_tree = cc.ContentNode(kind_id="topic", title="test", node_id="ccc") + self.channel.chef_tree = cc.ContentNode(kind_id=content_kinds.TOPIC, title="test", node_id="ccc") self.channel.chef_tree.save() self.channel.save() diff --git a/contentcuration/contentcuration/tests/test_urlendpoints.py b/contentcuration/contentcuration/tests/test_urlendpoints.py index d6f6f1d038..1d83092c46 100644 --- a/contentcuration/contentcuration/tests/test_urlendpoints.py +++ b/contentcuration/contentcuration/tests/test_urlendpoints.py @@ -8,9 +8,6 @@ class AllUrlsTest(StudioTestCase): - def setUp(self): - super(AllUrlsTest, self).setUp() - def test_responses(self, allowed_http_codes=None, default_kwargs=None, quiet=False): """ diff --git a/contentcuration/contentcuration/tests/testdata.py b/contentcuration/contentcuration/tests/testdata.py index 8c1e952e0a..3ddfa693e2 100644 --- a/contentcuration/contentcuration/tests/testdata.py +++ b/contentcuration/contentcuration/tests/testdata.py @@ -172,6 +172,7 @@ def channel(): def user(): user = cc.User.objects.create(email='user@test.com') user.set_password('password') + user.is_active = True user.save() return user diff --git a/contentcuration/contentcuration/utils/files.py b/contentcuration/contentcuration/utils/files.py index 86ed2df200..350211d1fd 100644 --- a/contentcuration/contentcuration/utils/files.py +++ b/contentcuration/contentcuration/utils/files.py @@ -23,7 +23,6 @@ def create_file_from_contents(contents, ext=None, node=None, preset_id=None, uploaded_by=None): checksum, _, path = write_raw_content_to_storage(contents, ext=ext) with default_storage.open(path, 'rb') as new_file: - # import ipdb; ipdb.set_trace() result = File.objects.create( file_on_disk=DjFile(new_file), file_format_id=ext, From 6c52f3a4c7a6ce1e0cf67d7465738cd237de0223 Mon Sep 17 00:00:00 2001 From: Micah Fitch Date: Fri, 7 Sep 2018 23:28:26 -0400 Subject: [PATCH 18/28] cleanup some debugging remnants --- contentcuration/contentcuration/serializers.py | 1 - .../contentcuration/static/js/bundle_modules/base.js | 2 +- .../contentcuration/static/js/edit_channel/models.js | 3 --- contentcuration/contentcuration/views/files.py | 1 - 4 files changed, 1 insertion(+), 6 deletions(-) diff --git a/contentcuration/contentcuration/serializers.py b/contentcuration/contentcuration/serializers.py index 7409c4480a..4d43ff77c2 100644 --- a/contentcuration/contentcuration/serializers.py +++ b/contentcuration/contentcuration/serializers.py @@ -123,7 +123,6 @@ class FileSerializer(BulkSerializerMixin, serializers.ModelSerializer): display_name = serializers.SerializerMethodField('retrieve_display_name') id = serializers.CharField(required=False) preset = FormatPresetSerializer(many=False, read_only=True) - # preset_id = serializers.PrimaryKeyRelatedField(many=False, queryset=FormatPreset.objects.all()) def get(*args, **kwargs): return super.get(*args, **kwargs) diff --git a/contentcuration/contentcuration/static/js/bundle_modules/base.js b/contentcuration/contentcuration/static/js/bundle_modules/base.js index 1a9343ebed..4cf6e3b553 100644 --- a/contentcuration/contentcuration/static/js/bundle_modules/base.js +++ b/contentcuration/contentcuration/static/js/bundle_modules/base.js @@ -6,7 +6,7 @@ require('../handlebars/helpers.js'); // side effect: adds shared styles to the DOM require('../../less/styles.less'); -// require('../utils/offline_helper'); +require('../utils/offline_helper'); // Promise polyfill if(!global.Promise) { diff --git a/contentcuration/contentcuration/static/js/edit_channel/models.js b/contentcuration/contentcuration/static/js/edit_channel/models.js index 71ea64ef77..32d7af5e3e 100644 --- a/contentcuration/contentcuration/static/js/edit_channel/models.js +++ b/contentcuration/contentcuration/static/js/edit_channel/models.js @@ -415,9 +415,6 @@ var ContentNodeCollection = BaseCollection.extend({ var to_add = new FileModel(file); var preset_data = to_add.get("preset"); preset_data.id = file.preset.name || file.preset.id; - // to_add.set('preset', preset_data); - // to_add.set('preset_id', preset_data['id']); - // to_add.set('contentnode', node.id); fileCollection.add(to_add); }); assessmentCollection.add(node.get('assessment_items')); diff --git a/contentcuration/contentcuration/views/files.py b/contentcuration/contentcuration/views/files.py index 554f7b3f9e..90ae903ea9 100644 --- a/contentcuration/contentcuration/views/files.py +++ b/contentcuration/contentcuration/views/files.py @@ -55,7 +55,6 @@ def file_create(request): if request.method != 'POST': return HttpResponseBadRequest("Only POST requests are allowed on this endpoint.") - # import ipdb; ipdb.set_trace() original_filename, ext = os.path.splitext(request.FILES.values()[0]._name) size = request.FILES.values()[0]._size contentfile = DjFile(request.FILES.values()[0]) From 70a300c8036a760c67cdd07560d3d7b92a56e55d Mon Sep 17 00:00:00 2001 From: Micah Fitch Date: Fri, 7 Sep 2018 23:45:34 -0400 Subject: [PATCH 19/28] cleanup one last debugging remnant -_- --- contentcuration/contentcuration/serializers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/contentcuration/contentcuration/serializers.py b/contentcuration/contentcuration/serializers.py index 4d43ff77c2..b89e623a03 100644 --- a/contentcuration/contentcuration/serializers.py +++ b/contentcuration/contentcuration/serializers.py @@ -75,7 +75,6 @@ def update(self, instance, validated_data): ret = [] update_files = {} user = self.context['request'].user - import ipdb with transaction.atomic(): # Get files that have the same contentnode, preset, and language as the files that are now attached to this node for item in validated_data: From 5b49affd90630825ffd035288786f00fb61eb347 Mon Sep 17 00:00:00 2001 From: Micah Fitch Date: Mon, 10 Sep 2018 00:15:10 -0400 Subject: [PATCH 20/28] get file format from extension --- contentcuration/contentcuration/models.py | 7 ++++- .../tests/test_garbage_collect.py | 3 -- .../contentcuration/tests/test_utils.py | 31 ++++++++++++++++++- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index 91c4b60f3a..1ecdf90a19 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -1070,7 +1070,12 @@ def save(self, *args, **kwargs): if not self.file_size: self.file_size = self.file_on_disk.size if not self.file_format: - self.file_format_id = os.path.splitext(self.file_on_disk.name)[1] + ext = os.path.splitext(self.file_on_disk.name)[1].lstrip('.') + try: + self.file_format = FileFormat.objects.get(pk=ext) + except ObjectDoesNotExist: + raise ValueError('Files of type `{}` are not supported.'.format(ext)) + super(File, self).save(*args, **kwargs) diff --git a/contentcuration/contentcuration/tests/test_garbage_collect.py b/contentcuration/contentcuration/tests/test_garbage_collect.py index 6860dce059..07ce0c585a 100755 --- a/contentcuration/contentcuration/tests/test_garbage_collect.py +++ b/contentcuration/contentcuration/tests/test_garbage_collect.py @@ -62,7 +62,6 @@ def test_deletes_associated_files(self): contentnode_id=c.pk, file_on_disk=ContentFile("test"), checksum="aaa", - file_format_id=file_formats.JPG ) f.file_on_disk.save("aaa.jpg", ContentFile("aaa")) @@ -149,12 +148,10 @@ def test_doesnt_delete_file_referenced_by_orphan_and_nonorphan_nodes(self): f = File.objects.create( contentnode=legit_node, checksum="aaa", - file_format_id=file_formats.JPG ) forphan = File.objects.create( contentnode=orphan_node, checksum="aaa", - file_format_id=file_formats.JPG ) # The file they both share. This has the same checksum and contents. diff --git a/contentcuration/contentcuration/tests/test_utils.py b/contentcuration/contentcuration/tests/test_utils.py index 42595fb621..fe9d89376b 100644 --- a/contentcuration/contentcuration/tests/test_utils.py +++ b/contentcuration/contentcuration/tests/test_utils.py @@ -2,10 +2,12 @@ from cStringIO import StringIO from django.core.files.storage import default_storage +from django.core.files.base import ContentFile + from django.conf import settings from unittest import TestCase -from contentcuration.models import User, generate_object_storage_name +from contentcuration.models import User, File, generate_object_storage_name from contentcuration.utils.policies import check_policies, POLICIES from contentcuration.utils.files import get_file_diff @@ -80,3 +82,30 @@ def test_returns_file_not_uploaded_yet(self): "rando" ] assert get_file_diff(files) == ["rando"] + +from contentcuration.models import FileFormat +class FileFormatsTestCase(StudioTestCase): + def test_unsupported_files_raise_error(self): + unsupported_file = File.objects.create( + file_on_disk=ContentFile("test"), + checksum='aaa' + ) + + try: + unsupported_file.file_on_disk.save("aaa.wtf", ContentFile("aaa")) + except ValueError as e: + pass + assert e + + def test_guess_format_from_extension(self): + known_extensions = [f.pk for f in FileFormat.objects.all()] + + for ext in known_extensions: + file_with_ext = File.objects.create( + file_on_disk=ContentFile("test"), + checksum="aaa" + ) + file_with_ext.file_on_disk.save("aaa.{}".format(ext), ContentFile("aaa")) + pass + + From 2b70d808db09a8eb6eb633c5a25fefe021fec4ab Mon Sep 17 00:00:00 2001 From: Micah Fitch Date: Mon, 10 Sep 2018 16:59:56 -0700 Subject: [PATCH 21/28] Don't query the database to guess file formats. Instead, use le_utils.constants.file_formats. --- .../migrations/0094_auto_20180910_2342.py | 25 ++++++++++++++++ contentcuration/contentcuration/models.py | 10 +++---- .../contentcuration/tests/test_utils.py | 18 +++++++++-- .../migrations/0004_auto_20180910_2342.py | 30 +++++++++++++++++++ 4 files changed, 75 insertions(+), 8 deletions(-) create mode 100644 contentcuration/contentcuration/migrations/0094_auto_20180910_2342.py create mode 100644 contentcuration/kolibri_content/migrations/0004_auto_20180910_2342.py diff --git a/contentcuration/contentcuration/migrations/0094_auto_20180910_2342.py b/contentcuration/contentcuration/migrations/0094_auto_20180910_2342.py new file mode 100644 index 0000000000..49c80b2316 --- /dev/null +++ b/contentcuration/contentcuration/migrations/0094_auto_20180910_2342.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.15 on 2018-09-10 23:42 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('contentcuration', '0093_auto_20180831_0745'), + ] + + operations = [ + migrations.AlterField( + model_name='fileformat', + name='extension', + field=models.CharField(choices=[(b'mp4', b'MP4 Video'), (b'vtt', b'VTT Subtitle'), (b'mp3', b'MP3 Audio'), (b'pdf', b'PDF Document'), (b'jpg', b'JPG Image'), (b'jpeg', b'JPEG Image'), (b'png', b'PNG Image'), (b'gif', b'GIF Image'), (b'json', b'JSON'), (b'svg', b'SVG Image'), (b'perseus', b'Perseus Exercise'), (b'graphie', b'Graphie Exercise'), (b'zip', b'HTML5 Zip'), (b'epub', b'ePub Document')], max_length=40, primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='formatpreset', + name='id', + field=models.CharField(choices=[(b'high_res_video', b'High Resolution'), (b'low_res_video', b'Low Resolution'), (b'vector_video', b'Vectorized'), (b'video_thumbnail', b'Thumbnail'), (b'video_subtitle', b'Subtitle'), (b'video_dependency', b'Video (dependency)'), (b'audio', b'Audio'), (b'audio_thumbnail', b'Thumbnail'), (b'document', b'Document'), (b'epub', b'ePub Document'), (b'document_thumbnail', b'Thumbnail'), (b'exercise', b'Exercise'), (b'exercise_thumbnail', b'Thumbnail'), (b'exercise_image', b'Exercise Image'), (b'exercise_graphie', b'Exercise Graphie'), (b'channel_thumbnail', b'Channel Thumbnail'), (b'topic_thumbnail', b'Thumbnail'), (b'html5_zip', b'HTML5 Zip'), (b'html5_dependency', b'HTML5 Dependency (Zip format)'), (b'html5_thumbnail', b'HTML5 Thumbnail')], max_length=150, primary_key=True, serialize=False), + ), + ] diff --git a/contentcuration/contentcuration/models.py b/contentcuration/contentcuration/models.py index 1ecdf90a19..d98aeb7056 100644 --- a/contentcuration/contentcuration/models.py +++ b/contentcuration/contentcuration/models.py @@ -1069,12 +1069,12 @@ def save(self, *args, **kwargs): self.checksum = md5.hexdigest() if not self.file_size: self.file_size = self.file_on_disk.size - if not self.file_format: + if not self.file_format_id: ext = os.path.splitext(self.file_on_disk.name)[1].lstrip('.') - try: - self.file_format = FileFormat.objects.get(pk=ext) - except ObjectDoesNotExist: - raise ValueError('Files of type `{}` are not supported.'.format(ext)) + if ext in dict(file_formats.choices).keys(): + self.file_format_id = ext + else: + raise ValueError("Files of type `{}` are not supported.".format(ext)) super(File, self).save(*args, **kwargs) diff --git a/contentcuration/contentcuration/tests/test_utils.py b/contentcuration/contentcuration/tests/test_utils.py index fe9d89376b..479be6b3f7 100644 --- a/contentcuration/contentcuration/tests/test_utils.py +++ b/contentcuration/contentcuration/tests/test_utils.py @@ -85,6 +85,9 @@ def test_returns_file_not_uploaded_yet(self): from contentcuration.models import FileFormat class FileFormatsTestCase(StudioTestCase): + """ + Ensure that unsupported files aren't saved. + """ def test_unsupported_files_raise_error(self): unsupported_file = File.objects.create( file_on_disk=ContentFile("test"), @@ -93,19 +96,28 @@ def test_unsupported_files_raise_error(self): try: unsupported_file.file_on_disk.save("aaa.wtf", ContentFile("aaa")) - except ValueError as e: + except Exception as e: pass assert e def test_guess_format_from_extension(self): - known_extensions = [f.pk for f in FileFormat.objects.all()] + """ + Make sure that we can guess file types listed in le_utils.file_formats.choices. + Note: if this test fails, it's likely because le_utils file formats aren't synced. + """ + from le_utils.constants import file_formats + known_extensions = dict(file_formats.choices).keys() for ext in known_extensions: file_with_ext = File.objects.create( file_on_disk=ContentFile("test"), checksum="aaa" ) - file_with_ext.file_on_disk.save("aaa.{}".format(ext), ContentFile("aaa")) + + try: + file_with_ext.file_on_disk.save("aaa.{}".format(ext), ContentFile("aaa")) + except Exception as e: + raise type(e)(e.message + " ... (hint: make sure that the version of le-utils you're using has its file formats synced).") pass diff --git a/contentcuration/kolibri_content/migrations/0004_auto_20180910_2342.py b/contentcuration/kolibri_content/migrations/0004_auto_20180910_2342.py new file mode 100644 index 0000000000..426e9119ad --- /dev/null +++ b/contentcuration/kolibri_content/migrations/0004_auto_20180910_2342.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.15 on 2018-09-10 23:42 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('content', '0003_contentnode_coach_content'), + ] + + operations = [ + migrations.AlterField( + model_name='file', + name='extension', + field=models.CharField(blank=True, choices=[(b'mp4', b'MP4 Video'), (b'vtt', b'VTT Subtitle'), (b'mp3', b'MP3 Audio'), (b'pdf', b'PDF Document'), (b'jpg', b'JPG Image'), (b'jpeg', b'JPEG Image'), (b'png', b'PNG Image'), (b'gif', b'GIF Image'), (b'json', b'JSON'), (b'svg', b'SVG Image'), (b'perseus', b'Perseus Exercise'), (b'graphie', b'Graphie Exercise'), (b'zip', b'HTML5 Zip'), (b'epub', b'ePub Document')], max_length=40), + ), + migrations.AlterField( + model_name='file', + name='preset', + field=models.CharField(blank=True, choices=[(b'high_res_video', b'High Resolution'), (b'low_res_video', b'Low Resolution'), (b'vector_video', b'Vectorized'), (b'video_thumbnail', b'Thumbnail'), (b'video_subtitle', b'Subtitle'), (b'video_dependency', b'Video (dependency)'), (b'audio', b'Audio'), (b'audio_thumbnail', b'Thumbnail'), (b'document', b'Document'), (b'epub', b'ePub Document'), (b'document_thumbnail', b'Thumbnail'), (b'exercise', b'Exercise'), (b'exercise_thumbnail', b'Thumbnail'), (b'exercise_image', b'Exercise Image'), (b'exercise_graphie', b'Exercise Graphie'), (b'channel_thumbnail', b'Channel Thumbnail'), (b'topic_thumbnail', b'Thumbnail'), (b'html5_zip', b'HTML5 Zip'), (b'html5_dependency', b'HTML5 Dependency (Zip format)'), (b'html5_thumbnail', b'HTML5 Thumbnail')], max_length=150), + ), + migrations.AlterField( + model_name='localfile', + name='extension', + field=models.CharField(blank=True, choices=[(b'mp4', b'MP4 Video'), (b'vtt', b'VTT Subtitle'), (b'mp3', b'MP3 Audio'), (b'pdf', b'PDF Document'), (b'jpg', b'JPG Image'), (b'jpeg', b'JPEG Image'), (b'png', b'PNG Image'), (b'gif', b'GIF Image'), (b'json', b'JSON'), (b'svg', b'SVG Image'), (b'perseus', b'Perseus Exercise'), (b'graphie', b'Graphie Exercise'), (b'zip', b'HTML5 Zip'), (b'epub', b'ePub Document')], max_length=40), + ), + ] From ff967e3cf04b4bb16292723090a4fa280b96c30e Mon Sep 17 00:00:00 2001 From: Micah Fitch Date: Mon, 10 Sep 2018 18:37:05 -0700 Subject: [PATCH 22/28] use in test --- contentcuration/contentcuration/tests/test_utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/contentcuration/contentcuration/tests/test_utils.py b/contentcuration/contentcuration/tests/test_utils.py index 479be6b3f7..b7a9feef97 100644 --- a/contentcuration/contentcuration/tests/test_utils.py +++ b/contentcuration/contentcuration/tests/test_utils.py @@ -94,11 +94,8 @@ def test_unsupported_files_raise_error(self): checksum='aaa' ) - try: + with self.assertRaises(Exception): unsupported_file.file_on_disk.save("aaa.wtf", ContentFile("aaa")) - except Exception as e: - pass - assert e def test_guess_format_from_extension(self): """ From 90bc5f0efcc7b5463d94f39282c234729e7f90ad Mon Sep 17 00:00:00 2001 From: Micah Fitch Date: Tue, 11 Sep 2018 14:48:33 -0700 Subject: [PATCH 23/28] Pipfile.lock --- Pipfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index e6406c0417..f20540939a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1131,7 +1131,7 @@ "sha256:9d98697f088eb1b0fa451391f91afb5e3ebde16bbdb272819fd091151fda4f1a", "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" ], - "markers": "python_version == '2.7'", + "markers": "python_version >= '2.6'", "version": "==1.5" }, "colorama": { @@ -1145,7 +1145,7 @@ "hashes": [ "sha256:5308b47021bc2340965c371f0f058cc6971a04502638d4244225c49d80db273a" ], - "markers": "python_version < '3.2'", + "markers": "python_version == '2.7'", "version": "==3.5.0" }, "django": { @@ -1207,7 +1207,7 @@ "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" ], - "markers": "python_version < '3.3'", + "markers": "python_version < '3.0'", "version": "==1.0.2" }, "future": { From 01d8ec52ea65376ac7d472206499042feaeef5e5 Mon Sep 17 00:00:00 2001 From: Micah Fitch Date: Mon, 17 Sep 2018 16:35:46 -0700 Subject: [PATCH 24/28] align auth_login/login URL route name to 'login' --- .../templates/registration/password_reset_complete.html | 2 +- contentcuration/contentcuration/urls.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contentcuration/contentcuration/templates/registration/password_reset_complete.html b/contentcuration/contentcuration/templates/registration/password_reset_complete.html index a06a87bcb1..86b1e96a1c 100644 --- a/contentcuration/contentcuration/templates/registration/password_reset_complete.html +++ b/contentcuration/contentcuration/templates/registration/password_reset_complete.html @@ -11,7 +11,7 @@

    {% trans "Password reset successfully" %}

    {% trans "Your password has been set. You may go ahead and log in now." %}

    -

    {% trans "Log in" %}

    +

    {% trans "Log in" %}

    {% endblock %} diff --git a/contentcuration/contentcuration/urls.py b/contentcuration/contentcuration/urls.py index 62d86d39c3..e1e48e55c0 100644 --- a/contentcuration/contentcuration/urls.py +++ b/contentcuration/contentcuration/urls.py @@ -237,7 +237,7 @@ def get_queryset(self): # Add account/registration endpoints urlpatterns += [ - url(r'^accounts/login/$', auth_views.login, {'template_name': 'registration/login.html'}, 'authentication_form': LoginForm, name='auth_login'), + url(r'^accounts/login/$', auth_views.login, {'template_name': 'registration/login.html', 'authentication_form': LoginForm}, name='login'), url(r'^accounts/logout/$', auth_views.logout, {'template_name': 'registration/logout.html'}, name='logout'), url( r'^accounts/password/reset/$', From eb6d81ff07e6dd6ef82cf65d6776ccce7b7a11d2 Mon Sep 17 00:00:00 2001 From: Micah Fitch Date: Tue, 18 Sep 2018 12:03:10 -0700 Subject: [PATCH 25/28] updated cloudbuild.yaml, courtesy of Aron --- cloudbuild.yaml | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 8c38eab3c9..9d85850500 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -20,7 +20,7 @@ steps: args: [ 'build', '-f', 'k8s/images/nginx/Dockerfile', - '--cache-from', 'gcr.io/$PROJECT_ID/learningequality-studio-nginx:latest', + '--cache-from', 'gcr.io/$PROJECT_ID/learningequality-studio-nginx:latest', '-t', 'gcr.io/$PROJECT_ID/learningequality-studio-nginx:$COMMIT_SHA', '-t', 'gcr.io/$PROJECT_ID/learningequality-studio-nginx:latest', '.' @@ -64,11 +64,22 @@ steps: - 'CLOUDSDK_CONTAINER_CLUSTER=dev-qa-cluster' secretEnv: ['POSTMARK_API_KEY'] entrypoint: 'bash' - args: [ - '-c', - # still run helm.bash to initialze creds, but otherwise run our own deploy script - '/builder/helm.bash && ./helm-deploy.sh $BRANCH_NAME studio-qa-content $COMMIT_SHA $$POSTMARK_API_KEY arontest $BRANCH_NAME arontest ops-central-studio-qa-sql-proxy-gcloud-sqlproxy.sqlproxy ../gcs-service-account.json' - ] + args: + - -c + - > + /builder/helm.bash && + ./helm-deploy.sh + $BRANCH_NAME + studio-qa-content + $COMMIT_SHA + $$POSTMARK_API_KEY + arontest + $BRANCH_NAME + arontest + $PROJECT_ID-studio-qa-sql-proxy-gcloud-sqlproxy.sqlproxy + ../gcs-service-account.json + $PROJECT_ID + is_production timeout: 3600s secrets: @@ -81,3 +92,4 @@ images: - 'gcr.io/$PROJECT_ID/learningequality-studio-nginx:latest' - 'gcr.io/$PROJECT_ID/learningequality-studio-app:$COMMIT_SHA' - 'gcr.io/$PROJECT_ID/learningequality-studio-app:latest' + From 89b23c1b22a06e06e29f449568dc2cfc6cc535c1 Mon Sep 17 00:00:00 2001 From: Aron Fyodor Asor <191955+aronasorman@users.noreply.github.com> Date: Thu, 20 Sep 2018 10:40:17 -0700 Subject: [PATCH 26/28] Create simplified implementation for gcs_storage.generate_filename Just return the filename passed in, since the logic is implemented through generate_object_storage_name. --- contentcuration/contentcuration/utils/gcs_storage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contentcuration/contentcuration/utils/gcs_storage.py b/contentcuration/contentcuration/utils/gcs_storage.py index 91ff74ed8d..7ac1ceeb4c 100644 --- a/contentcuration/contentcuration/utils/gcs_storage.py +++ b/contentcuration/contentcuration/utils/gcs_storage.py @@ -145,4 +145,5 @@ def get_valid_name(self, name): return name def generate_filename(self, filename): - raise NotImplementedError + # TODO(aron): can we move the generate_object_storage_name logic to here? + return filename From a04bcec32dd5fb1bb88812deea79084eb47cfe72 Mon Sep 17 00:00:00 2001 From: Aron Fyodor Asor <191955+aronasorman@users.noreply.github.com> Date: Fri, 14 Sep 2018 08:58:43 -0700 Subject: [PATCH 27/28] Remove assert about the filename and a string --- contentcuration/contentcuration/utils/gcs_storage.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/contentcuration/contentcuration/utils/gcs_storage.py b/contentcuration/contentcuration/utils/gcs_storage.py index 7ac1ceeb4c..f38e208d2c 100644 --- a/contentcuration/contentcuration/utils/gcs_storage.py +++ b/contentcuration/contentcuration/utils/gcs_storage.py @@ -28,8 +28,6 @@ def _determine_content_type(cls, filename): Raises an AssertionError if filename is not a string. """ - assert isinstance(filename, str), "Expected filename to be string, passed in {}".format(filename) - typ, _ = mimetypes.guess_type(filename) if not typ: From afa4c10791c7532a20bb0d9c305bc542cfadb0fc Mon Sep 17 00:00:00 2001 From: kollivier Date: Thu, 27 Sep 2018 07:35:37 -0700 Subject: [PATCH 28/28] Partial fix for #975. Restore the Other option textbox in the reg info form. --- .../templates/registration/registration_information_form.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentcuration/contentcuration/templates/registration/registration_information_form.html b/contentcuration/contentcuration/templates/registration/registration_information_form.html index cb2f26e0b5..bd6be6dfc0 100644 --- a/contentcuration/contentcuration/templates/registration/registration_information_form.html +++ b/contentcuration/contentcuration/templates/registration/registration_information_form.html @@ -38,7 +38,7 @@

    {% trans 'Create an Account' %}

    {{form.storage.label}}*  {{form.storage}}
    {% endif %} - {% if option.choice_value == 'other' %} + {% if option.data.value == 'other' %} {{form.other_use}} {% endif %}