From e4fef1dab0c339d5bf5f75d4beeddafdf5ba0947 Mon Sep 17 00:00:00 2001 From: jingma Date: Mon, 6 Jul 2020 02:06:07 +0200 Subject: [PATCH 1/3] 7056 Skip unknown types in a dataset which cause the whole dataset to fail from being harvested --- .../iq/dataverse/util/json/JsonParser.java | 30 +++------- .../dataverse/util/json/JsonParserTest.java | 56 +++++-------------- 2 files changed, 22 insertions(+), 64 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index 144725c2f81..565af9779e4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -1,23 +1,7 @@ package edu.harvard.iq.dataverse.util.json; import com.google.gson.Gson; -import edu.harvard.iq.dataverse.ControlledVocabularyValue; -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.DataFileCategory; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetField; -import edu.harvard.iq.dataverse.DatasetFieldConstant; -import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; -import edu.harvard.iq.dataverse.DatasetFieldServiceBean; -import edu.harvard.iq.dataverse.DatasetFieldType; -import edu.harvard.iq.dataverse.DatasetFieldValue; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DataverseContact; -import edu.harvard.iq.dataverse.DataverseTheme; -import edu.harvard.iq.dataverse.FileMetadata; -import edu.harvard.iq.dataverse.MetadataBlockServiceBean; -import edu.harvard.iq.dataverse.TermsOfUseAndAccess; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.TermsOfUseAndAccess.License; import edu.harvard.iq.dataverse.api.Util; import edu.harvard.iq.dataverse.api.dto.FieldDTO; @@ -32,14 +16,14 @@ import edu.harvard.iq.dataverse.workflow.step.WorkflowStepData; import org.apache.commons.validator.routines.DomainValidator; +import javax.json.*; +import javax.json.JsonValue.ValueType; import java.io.StringReader; import java.sql.Timestamp; import java.text.ParseException; import java.util.*; import java.util.logging.Logger; import java.util.stream.Collectors; -import javax.json.*; -import javax.json.JsonValue.ValueType; /** * Parses JSON objects into domain objects. @@ -394,7 +378,10 @@ private List parseFieldsFromArray(JsonArray fieldsArray, Boolean t List fields = new LinkedList<>(); for (JsonObject fieldJson : fieldsArray.getValuesAs(JsonObject.class)) { try { - fields.add(parseField(fieldJson, testType)); + DatasetField field = parseField(fieldJson, testType); + if (field != null) { + fields.add(field); + } } catch (CompoundVocabularyException ex) { DatasetFieldType fieldType = datasetFieldSvc.findByNameOpt(fieldJson.getString("typeName", "")); if (lenient && (DatasetFieldConstant.geographicCoverage).equals(fieldType.getName())) { @@ -566,7 +553,8 @@ public DatasetField parseField(JsonObject json, Boolean testType) throws JsonPar if (type == null) { - throw new JsonParseException("Can't find type '" + json.getString("typeName", "") + "'"); + logger.info("Can't find type '" + json.getString("typeName", "") + "'"); + return null; } if (testType && type.isAllowMultiples() != json.getBoolean("multiple")) { throw new JsonParseException("incorrect multiple for field " + json.getString("typeName", "")); diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java index 9b452c464b3..f9c37783a84 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java @@ -4,62 +4,32 @@ package edu.harvard.iq.dataverse.util.json; -import edu.harvard.iq.dataverse.ControlledVocabularyValue; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetField; -import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; -import edu.harvard.iq.dataverse.DatasetFieldServiceBean; -import edu.harvard.iq.dataverse.DatasetFieldType; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.DatasetFieldType.FieldType; -import edu.harvard.iq.dataverse.DatasetFieldValue; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseTheme.Alignment; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.IpGroup; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.IpGroupProvider; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddressRange; -import edu.harvard.iq.dataverse.DataverseTheme.Alignment; -import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.authorization.groups.impl.maildomain.MailDomainGroup; import edu.harvard.iq.dataverse.authorization.groups.impl.maildomain.MailDomainGroupTest; import edu.harvard.iq.dataverse.authorization.users.GuestUser; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.mocks.MockDatasetFieldSvc; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.StringReader; -import java.math.BigDecimal; -import java.text.ParseException; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TimeZone; -import javax.json.Json; -import javax.json.JsonArray; -import javax.json.JsonArrayBuilder; -import javax.json.JsonObject; -import javax.json.JsonObjectBuilder; -import javax.json.JsonReader; -import javax.json.JsonValue; import org.junit.AfterClass; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import javax.json.*; +import java.io.*; +import java.math.BigDecimal; +import java.text.ParseException; +import java.util.*; + +import static org.junit.Assert.*; + /** * * @author michael @@ -433,12 +403,12 @@ public void testParseEmptyDataset() throws JsonParseException { /** * - * Expect an exception when the dataset version JSON contains fields + * Expect no exception when the dataset version JSON contains fields * that the {@link DatasetFieldService} doesn't know about. - * @throws JsonParseException as expected + * @throws JsonParseException should not happen here * @throws IOException when test file IO goes wrong - this is bad. */ - @Test(expected = JsonParseException.class) + @Test public void testParseOvercompleteDatasetVersion() throws JsonParseException, IOException { JsonObject dsJson; try (InputStream jsonFile = ClassLoader.getSystemResourceAsStream("json/complete-dataset-version.json")) { From 1e0ea12d77393cfc765d8b2233cc49b844ab9819 Mon Sep 17 00:00:00 2001 From: jingma Date: Fri, 10 Jul 2020 07:00:19 +0200 Subject: [PATCH 2/3] 4597 Change log level to fine. --- .../iq/dataverse/util/json/JsonParser.java | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index 565af9779e4..0f908accdb4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -1,7 +1,23 @@ package edu.harvard.iq.dataverse.util.json; import com.google.gson.Gson; -import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.ControlledVocabularyValue; +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.DataFileCategory; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetField; +import edu.harvard.iq.dataverse.DatasetFieldConstant; +import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; +import edu.harvard.iq.dataverse.DatasetFieldServiceBean; +import edu.harvard.iq.dataverse.DatasetFieldType; +import edu.harvard.iq.dataverse.DatasetFieldValue; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseContact; +import edu.harvard.iq.dataverse.DataverseTheme; +import edu.harvard.iq.dataverse.FileMetadata; +import edu.harvard.iq.dataverse.MetadataBlockServiceBean; +import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.TermsOfUseAndAccess.License; import edu.harvard.iq.dataverse.api.Util; import edu.harvard.iq.dataverse.api.dto.FieldDTO; @@ -16,14 +32,14 @@ import edu.harvard.iq.dataverse.workflow.step.WorkflowStepData; import org.apache.commons.validator.routines.DomainValidator; -import javax.json.*; -import javax.json.JsonValue.ValueType; import java.io.StringReader; import java.sql.Timestamp; import java.text.ParseException; import java.util.*; import java.util.logging.Logger; import java.util.stream.Collectors; +import javax.json.*; +import javax.json.JsonValue.ValueType; /** * Parses JSON objects into domain objects. @@ -553,7 +569,7 @@ public DatasetField parseField(JsonObject json, Boolean testType) throws JsonPar if (type == null) { - logger.info("Can't find type '" + json.getString("typeName", "") + "'"); + logger.fine("Can't find type '" + json.getString("typeName", "") + "'"); return null; } if (testType && type.isAllowMultiples() != json.getBoolean("multiple")) { From 057a8af4935817c551b3c05739ff3a769704e552 Mon Sep 17 00:00:00 2001 From: jingma Date: Fri, 10 Jul 2020 07:06:55 +0200 Subject: [PATCH 3/3] 4597 Change back to full import statements. --- .../dataverse/util/json/JsonParserTest.java | 50 +++++++++++++++---- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java index f9c37783a84..005b33a8bcb 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonParserTest.java @@ -4,32 +4,62 @@ package edu.harvard.iq.dataverse.util.json; -import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.ControlledVocabularyValue; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetField; +import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; +import edu.harvard.iq.dataverse.DatasetFieldServiceBean; +import edu.harvard.iq.dataverse.DatasetFieldType; import edu.harvard.iq.dataverse.DatasetFieldType.FieldType; -import edu.harvard.iq.dataverse.DataverseTheme.Alignment; +import edu.harvard.iq.dataverse.DatasetFieldValue; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.IpGroup; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.IpGroupProvider; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddressRange; +import edu.harvard.iq.dataverse.DataverseTheme.Alignment; +import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.authorization.groups.impl.maildomain.MailDomainGroup; import edu.harvard.iq.dataverse.authorization.groups.impl.maildomain.MailDomainGroupTest; import edu.harvard.iq.dataverse.authorization.users.GuestUser; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.mocks.MockDatasetFieldSvc; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.StringReader; +import java.math.BigDecimal; +import java.text.ParseException; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; +import javax.json.Json; +import javax.json.JsonArray; +import javax.json.JsonArrayBuilder; +import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; +import javax.json.JsonReader; +import javax.json.JsonValue; import org.junit.AfterClass; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import javax.json.*; -import java.io.*; -import java.math.BigDecimal; -import java.text.ParseException; -import java.util.*; - -import static org.junit.Assert.*; - /** * * @author michael