From 9d116cadd444d59df8fc261c636fb17eca366035 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 Jul 2022 10:13:04 +0000 Subject: [PATCH 1/5] chore(deps): bump jts.version from 1.18.2 to 1.19.0 in /dhis-2 Bumps `jts.version` from 1.18.2 to 1.19.0. Updates `jts-core` from 1.18.2 to 1.19.0 Updates `jts-io-common` from 1.18.2 to 1.19.0 --- updated-dependencies: - dependency-name: org.locationtech.jts:jts-core dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: org.locationtech.jts.io:jts-io-common dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- dhis-2/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/pom.xml b/dhis-2/pom.xml index 971b41e193d4..0989dcbf776b 100644 --- a/dhis-2/pom.xml +++ b/dhis-2/pom.xml @@ -138,7 +138,7 @@ 2.7 1.5.2.Final 27.0 - 1.18.2 + 1.19.0 2.23.1 From 22cc1ed64e4b5e983e122487f65a9a3cefc1bfcd Mon Sep 17 00:00:00 2001 From: teleivo Date: Thu, 28 Jul 2022 14:14:19 +0200 Subject: [PATCH 2/5] fix: error parsing event geometry during CSV import --- .../hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java | 2 +- .../webapi/controller/tracker/csv/TrackerCsvEventService.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java index fa7db37a9392..e79d03b94666 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java @@ -148,7 +148,7 @@ public List readEvents( InputStream inputStream, boolean skipFirst ) event.setCompletedDate( dataValue.getCompletedDate() ); event.setCompletedBy( dataValue.getCompletedBy() ); - if ( dataValue.getGeometry() != null ) + if ( StringUtils.isNotBlank( dataValue.getGeometry() ) ) { event.setGeometry( new WKTReader().read( dataValue.getGeometry() ) ); } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/csv/TrackerCsvEventService.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/csv/TrackerCsvEventService.java index 53bc5418a500..89800f214ed8 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/csv/TrackerCsvEventService.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/csv/TrackerCsvEventService.java @@ -193,7 +193,7 @@ public List readEvents( InputStream inputStream, boolean skipFirst ) event.setAttributeCategoryOptions( dataValue.getAttributeCategoryOptions() ); event.setAssignedUser( User.builder().username( dataValue.getAssignedUser() ).build() ); - if ( dataValue.getGeometry() != null ) + if ( StringUtils.isNotBlank( dataValue.getGeometry() ) ) { event.setGeometry( new WKTReader().read( dataValue.getGeometry() ) ); } From ab3c2af7d8672acbfd0e0f236c0dcc48fe835c51 Mon Sep 17 00:00:00 2001 From: teleivo Date: Fri, 29 Jul 2022 12:49:34 +0200 Subject: [PATCH 3/5] fix: parsing geometry with leading single quotes as locationtech/jts parser has either gotten stricter or not reported such parser errors before https://github.com/locationtech/jts/pull/786 --- .../event/csv/DefaultCsvEventService.java | 6 +++- dhis-2/dhis-web-api/pom.xml | 5 +++ .../tracker/csv/TrackerCsvEventService.java | 6 +++- .../csv/TrackerCsvEventServiceTest.java | 31 +++++++++++++++++++ 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java index e79d03b94666..616641fd6f19 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java @@ -32,6 +32,7 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.List; +import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; import org.hisp.dhis.dxf2.events.event.DataValue; @@ -61,6 +62,8 @@ public class DefaultCsvEventService private static final CsvSchema CSV_SCHEMA = CSV_MAPPER.schemaFor( CsvEventDataValue.class ) .withLineSeparator( "\n" ); + private static final Pattern TRIM_SINGLE_QUOTES = Pattern.compile( "^'|'$" ); + @Override public void writeEvents( OutputStream outputStream, List events, boolean withHeader ) throws IOException @@ -150,7 +153,8 @@ public List readEvents( InputStream inputStream, boolean skipFirst ) if ( StringUtils.isNotBlank( dataValue.getGeometry() ) ) { - event.setGeometry( new WKTReader().read( dataValue.getGeometry() ) ); + event.setGeometry( new WKTReader() + .read( TRIM_SINGLE_QUOTES.matcher( dataValue.getGeometry() ).replaceAll( "" ) ) ); } else if ( dataValue.getLongitude() != null && dataValue.getLatitude() != null ) { diff --git a/dhis-2/dhis-web-api/pom.xml b/dhis-2/dhis-web-api/pom.xml index 9c771b59f0aa..c8b3bf9097b7 100644 --- a/dhis-2/dhis-web-api/pom.xml +++ b/dhis-2/dhis-web-api/pom.xml @@ -326,6 +326,11 @@ junit-jupiter test + + org.junit.jupiter + junit-jupiter-params + test + org.hisp.dhis dhis-support-test diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/csv/TrackerCsvEventService.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/csv/TrackerCsvEventService.java index 89800f214ed8..0e88c9f8d710 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/csv/TrackerCsvEventService.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/csv/TrackerCsvEventService.java @@ -33,6 +33,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.regex.Pattern; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -63,6 +64,8 @@ public class TrackerCsvEventService { private static final CsvMapper CSV_MAPPER = new CsvMapper().enable( CsvParser.Feature.WRAP_AS_ARRAY ); + private static final Pattern TRIM_SINGLE_QUOTES = Pattern.compile( "^'|'$" ); + @Override public void writeEvents( OutputStream outputStream, List events, boolean withHeader ) throws IOException @@ -195,7 +198,8 @@ public List readEvents( InputStream inputStream, boolean skipFirst ) if ( StringUtils.isNotBlank( dataValue.getGeometry() ) ) { - event.setGeometry( new WKTReader().read( dataValue.getGeometry() ) ); + event.setGeometry( new WKTReader() + .read( TRIM_SINGLE_QUOTES.matcher( dataValue.getGeometry() ).replaceAll( "" ) ) ); } else if ( dataValue.getLongitude() != null && dataValue.getLatitude() != null ) { diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/csv/TrackerCsvEventServiceTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/csv/TrackerCsvEventServiceTest.java index 89c804172abb..5f0dd8ddd6ac 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/csv/TrackerCsvEventServiceTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/csv/TrackerCsvEventServiceTest.java @@ -32,10 +32,12 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.List; import java.util.Set; @@ -46,6 +48,11 @@ import org.hisp.dhis.webapi.controller.tracker.view.User; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.Point; import org.locationtech.jts.io.ParseException; import com.google.common.io.Files; @@ -55,10 +62,13 @@ class TrackerCsvEventServiceTest private TrackerCsvEventService service; + private GeometryFactory geometryFactory; + @BeforeEach void setUp() { service = new TrackerCsvEventService(); + geometryFactory = new GeometryFactory(); } @Test @@ -209,4 +219,25 @@ void testReadEventsFromFileWithoutHeader() assertFalse( dv.isProvidedElsewhere() ); } ); } + + @ValueSource( strings = { + ",,,,,,,,,POINT (-11.4283223849698 8.06311527044516)", + ",,,,,,,,,\"POINT (-11.4283223849698 8.06311527044516)\"", + ",,,,,,,,,'POINT (-11.4283223849698 8.06311527044516)'", + } ) + @ParameterizedTest + void testReadEventsParsesGeometryEvenIfQuoted( String csv ) + throws IOException, + ParseException + { + + InputStream in = new ByteArrayInputStream( csv.getBytes( StandardCharsets.UTF_8 ) ); + + List events = service.readEvents( in, false ); + + assertFalse( events.isEmpty() ); + assertEquals( 1, events.size() ); + Point expected = geometryFactory.createPoint( new Coordinate( -11.4283223849698, 8.06311527044516 ) ); + assertEquals( expected, events.get( 0 ).getGeometry() ); + } } From 6f3c72cca0aa01d06e1167905ce98ef8a777d4ec Mon Sep 17 00:00:00 2001 From: teleivo Date: Fri, 29 Jul 2022 13:43:00 +0200 Subject: [PATCH 4/5] fix: sonarqube suggestion Alternatives in regular expressions should be grouped when used with anchors --- .../hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java | 2 +- .../webapi/controller/tracker/csv/TrackerCsvEventService.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java index 616641fd6f19..ed063c7ef2b2 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java @@ -62,7 +62,7 @@ public class DefaultCsvEventService private static final CsvSchema CSV_SCHEMA = CSV_MAPPER.schemaFor( CsvEventDataValue.class ) .withLineSeparator( "\n" ); - private static final Pattern TRIM_SINGLE_QUOTES = Pattern.compile( "^'|'$" ); + private static final Pattern TRIM_SINGLE_QUOTES = Pattern.compile( "(?:^')|(?:'$)" ); @Override public void writeEvents( OutputStream outputStream, List events, boolean withHeader ) diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/csv/TrackerCsvEventService.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/csv/TrackerCsvEventService.java index 0e88c9f8d710..dc6ccaa87e81 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/csv/TrackerCsvEventService.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/csv/TrackerCsvEventService.java @@ -64,7 +64,7 @@ public class TrackerCsvEventService { private static final CsvMapper CSV_MAPPER = new CsvMapper().enable( CsvParser.Feature.WRAP_AS_ARRAY ); - private static final Pattern TRIM_SINGLE_QUOTES = Pattern.compile( "^'|'$" ); + private static final Pattern TRIM_SINGLE_QUOTES = Pattern.compile( "(?:^')|(?:'$)" ); @Override public void writeEvents( OutputStream outputStream, List events, boolean withHeader ) From 7e6e3d07a8f62829319a50fb2076437e807be82f Mon Sep 17 00:00:00 2001 From: teleivo Date: Fri, 29 Jul 2022 13:59:53 +0200 Subject: [PATCH 5/5] Revert "fix: sonarqube suggestion" This reverts commit 6f3c72cca0aa01d06e1167905ce98ef8a777d4ec. as Sonarqube is now saying that I should remove these unecessary groups --- .../hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java | 2 +- .../webapi/controller/tracker/csv/TrackerCsvEventService.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java index ed063c7ef2b2..616641fd6f19 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/events/event/csv/DefaultCsvEventService.java @@ -62,7 +62,7 @@ public class DefaultCsvEventService private static final CsvSchema CSV_SCHEMA = CSV_MAPPER.schemaFor( CsvEventDataValue.class ) .withLineSeparator( "\n" ); - private static final Pattern TRIM_SINGLE_QUOTES = Pattern.compile( "(?:^')|(?:'$)" ); + private static final Pattern TRIM_SINGLE_QUOTES = Pattern.compile( "^'|'$" ); @Override public void writeEvents( OutputStream outputStream, List events, boolean withHeader ) diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/csv/TrackerCsvEventService.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/csv/TrackerCsvEventService.java index dc6ccaa87e81..0e88c9f8d710 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/csv/TrackerCsvEventService.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/csv/TrackerCsvEventService.java @@ -64,7 +64,7 @@ public class TrackerCsvEventService { private static final CsvMapper CSV_MAPPER = new CsvMapper().enable( CsvParser.Feature.WRAP_AS_ARRAY ); - private static final Pattern TRIM_SINGLE_QUOTES = Pattern.compile( "(?:^')|(?:'$)" ); + private static final Pattern TRIM_SINGLE_QUOTES = Pattern.compile( "^'|'$" ); @Override public void writeEvents( OutputStream outputStream, List events, boolean withHeader )