full implementation of functional test

doajtest/example_files/publisher_csv_upload_success.csv

@@ -0,0 +1,4 @@
+Journal title,Journal ISSN (print version),Journal EISSN (online version),Publisher,APC,APC information URL,APC amount,Journal waiver policy (for developing country authors etc),Waiver policy information URL,Has other fees,Other fees information URL (only if answer is in Column J is 'Yes')
+Electronics Letters,0000-0001,0000-0002,Another,Yes,https://ietresearch.onlinelibrary.wiley.com/hub/journal/1350911x/homepage/open-access,2200 USD; 1700 GBP; 2000 EUR,Yes,https://authorservices.wiley.com/open-research/open-access/for-authors/waivers-and-discounts.html,No,
+,,,,,,,,,,
+Journal of Veterinary Internal Medicine,0000-0003,,Another,Yes,http://onlinelibrary.wiley.com/journal/10.1111/%28ISSN%291939-1676/homepage/article_publication_charges.htm,1650 USD,Yes,http://www.wileyopenaccess.com/details/content/13707a1ddf6/Waivers-and-Discounts-on-Article-Publication-Charges.html,No,http://onlinelibrary.wiley.com/journal/10.1111/%28ISSN%291939-1676/homepage/article_publication_charges.htm

doajtest/example_files/publisher_csv_upload_warnings.csv

@@ -0,0 +1,5 @@
+journal title,Journal ISSN (print version),Journal EISSN (online version),Publisher,APC,APC information URL,APC amount,Journal waiver policy (for developing country authors etc),Waiver policy information URL,Has other fees,Other fees information URL (only if answer is in Column J is 'Yes')
+Electronics Letters,0000-0001,0000-0002,Another,Yes,https://ietresearch.onlinelibrary.wiley.com/hub/journal/1350911x/homepage/open-access,2200 USD; 1700 GBP; 2000 EUR,Yes,https://authorservices.wiley.com/open-research/open-access/for-authors/waivers-and-discounts.html,No,
+,,,,,,,,,,
+Journal of Veterinary Internal Medicine,0000-0003,,Another,Yes,http://onlinelibrary.wiley.com/journal/10.1111/%28ISSN%291939-1676/homepage/article_publication_charges.htm,1650 USD,Yes,http://www.wileyopenaccess.com/details/content/13707a1ddf6/Waivers-and-Discounts-on-Article-Publication-Charges.html,No,http://onlinelibrary.wiley.com/journal/10.1111/%28ISSN%291939-1676/homepage/article_publication_charges.htm
+No Updates,,0000-0010,,,,,,,,

doajtest/testbook/publisher_csv/validate_csv.yml

@@ -0,0 +1,79 @@
+suite: Publisher CSV
+testset: Validate CSV
+tests:
+- title: Invalid headers in upload
+  context:
+    role: Publisher
+    testdrive: publisher_csv_upload
+  setup:
+    - Use the publisher_csv_upload testdrive to setup for this test at /testdrive/publisher_csv_upload
+    - At the end of the test please use the 'teardown' link provided by the testdrive to remove test assets from the system
+  steps:
+  - step: Log in as the publisher account specified by the testdrive result
+  - step: Go to the 'Validate your Journal CSV' tab in the publisher dashboard
+    path: publisher/journal-csv
+    results:
+    - There is a page that offers you documentation and an upload box for your Journal CSV
+  - step: Click 'Choose File' and select a CSV file with invalid headers
+    resource: /doajtest/example_files/publisher_csv_invalid_headers.csv
+  - step: Click 'Validate'
+    results:
+      - You are briefly shown a message to say that your file is being validated (this may appear only briefly, if you 
+        blink you may miss it)
+      - Shortly after, you are shown a set of validation errors.  The following are the expected errors
+      - A general error saying that "Journal ISSN (print version)" is a required header and is missing
+      - An issue with the header row in Column A, warning that the header case is different from the expected header
+      - An issue with the header row in Column K, saying "Extra unexpected header" is not a valid header
+  - step: If you are finished testing, you can teardown the testdrive using the link provided.  If you are going to continue 
+      with the other tests in this suite you can leave the testdrive in place until the very end
+
+- title: Invalid data in upload
+  context:
+    role: Publisher
+    testdrive: publisher_csv_upload
+  setup:
+    - If you have not done so already, use the publisher_csv_upload testdrive to setup for this test at /testdrive/publisher_csv_upload
+  steps:
+    - step: Log in as the publisher account specified by the testdrive result
+    - step: Go to the 'Validate your Journal CSV' tab in the publisher dashboard
+      path: publisher/journal-csv
+    - step: Click 'Choose File' and select a CSV file with invalid data
+      resource: /doajtest/example_files/publisher_csv_invalid_data.csv
+    - step: Click 'Validate'
+      results:
+        - You are shown a set of validation errors.  The following are the expected errors
+        - Row 2 and Row 6 were not found in DOAJ
+        - Row 3 is owned by another user
+        - Row 4 did not provide any updates to the journal
+        - Row 5 changed the journal title, which is not permitted
+        - Row 7 had a selection of validation errors.  The errors are identified, and the original and newly supplied
+          values are shown
+    - step: If you are finished testing, you can teardown the testdrive using the link provided.  If you are going to continue
+        with the other tests in this suite you can leave the testdrive in place until the very end
+
+- title: Valid uploads
+  context:
+    role: Publisher
+    testdrive: publisher_csv_upload
+  setup:
+    - If you have not done so already, use the publisher_csv_upload testdrive to setup for this test at /testdrive/publisher_csv_upload
+  steps:
+    - step: Log in as the publisher account specified by the testdrive result
+    - step: Go to the 'Validate your Journal CSV' tab in the publisher dashboard
+      path: publisher/journal-csv
+    - step: Click 'Validate' without selecting a file
+      results:
+        - You are shown a message saying that you must select a file to upload
+    - step: Click 'Choose File' and select a CSV file with valid data that will generate warnings
+      resource: /doajtest/example_files/publisher_csv_upload_warnings.csv
+    - step: Click 'Validate'
+      results:
+        - You are shown a validation message that says validation is successful with warnings
+        - You are shown a list of warnings.  The following are the expected warnings
+        - One of the headers is in the wrong case
+        - One of the rows does not provide an update to the data
+    - step: Click 'Choose File' and select a CSV file with valid data that will not generate any warnings
+      resource: /doajtest/example_files/publisher_csv_upload_success.csv
+    - step: Click 'Validate'
+      results:
+        - You are shown a validation message that says validation is successful

portality/bll/services/application.py

@@ -580,14 +580,15 @@ def validate_update_csv(self, file_path, account: models.Account):
             # verify header row with current CSV headers, report errors
             header_row = reader.fieldnames
             allowed_headers = Journal2PublisherUploadQuestionsXwalk.question_list()
-            lower_case_allowed_headers = map(str.lower, allowed_headers)
+            lower_case_allowed_headers = list(map(str.lower, allowed_headers))
             required_headers = Journal2PublisherUploadQuestionsXwalk.required_questions()
 
             # Always perform a match check on supplied headers, not counting order
             for i, h in enumerate(header_row):
                 if h and h not in allowed_headers:
                     if h.lower() in lower_case_allowed_headers:
-                        validation.header(validation.WARN, i, f'"{h}" has mismatching case to expected header "{allowed_headers[i]}".')
+                        expected = allowed_headers[lower_case_allowed_headers.index(h.lower())]
+                        validation.header(validation.WARN, i, f'"{h}" has mismatching case to expected header "{expected}".')
                     else:
                         validation.header(validation.ERROR, i, f'"{h}" is not a valid header.')
 
@@ -766,6 +767,8 @@ def _cleanhtml(self, raw_html):
 
     def json(self, indent=None):
         repr = {
+            "has_errors": self._errors,
+            "has_warnings": self._warnings,
             "general": self._general,
             "headers": self._headers,
             "rows": self._row,

portality/static/js/publisher_csv.js

@@ -3,12 +3,22 @@ doaj.publisher_csvs = {};
 doaj.publisher_csvs.init = function() {
     $("#validate").on("click", function(event) {
         event.preventDefault();
-        $("#validation-results").html("Validating your file, please wait ...");
 
         let fd = new FormData();
         let file = $('#upload-csv-file')[0].files[0];
+        if (!file) {
+            alert("You must select a file to validate");
+            return;
+        }
+        if (!file.name.endsWith(".csv")) {
+            alert("You must provide a CSV file");
+            return;
+        }
+
         fd.append("journal_csv", file);
 
+        $("#validation-results").html("Validating your file, please wait ...");
+
         $.ajax({
            url: "/publisher/journal-csv/validate",
            type: "POST",
@@ -72,8 +82,12 @@ doaj.publisher_csvs.render_validation_results = function(response) {
     }
 
     let successFrag = "";
-    if (generalFrag === "" && headerFrag === "" && rowsFrag === "" && valuesFrag === "") {
-        successFrag = "<h3>File validated successfully!</h3>You can now go ahead and send this file to us for processing.";
+    if (!response.has_errors) {
+        if (response.has_warnings) {
+            successFrag = "<h3>File validated with warnings!</h3>You can now go ahead and send this file to us for processing, but please check the warnings below.";
+        } else {
+            successFrag = "<h3>File validated successfully!</h3>You can now go ahead and send this file to us for processing.";
+        }
     }
 
     let frag = successFrag + generalFrag + headerFrag + rowsFrag + valuesFrag;

portality/templates/publisher/nav.html

@@ -14,7 +14,7 @@
     (xml, "Upload article XML", 3),
     (metadata, "Enter article metadata", 4),
     (preservation, "Upload preservation file", 5),
-    (csv, "Upload journal CSV", 6),
+    (csv, "Validate your Journal CSV", 6),
     (help, "Help", 7),
     ]
 %}

portality/view/publisher.py

@@ -382,8 +382,8 @@ def journal_csv_validate():
     if "journal_csv" not in request.files:
         abort(400)
     file = request.files["journal_csv"]
-    # if file.size() > app.config.get("JOURNAL_CSV_UPLOAD__MAX_FILE_SIZE", 1000000):
-    #     abort(400)
+    if not file.filename.endswith(".csv"):
+        abort(400)
 
     tmpStore = StoreFactory.tmp()
     container_id = app.config.get("JOURNAL_CSV_UPLOAD__TMP_CONTAINER", "publisher_csvs_validation")