Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Log and Return Failed Sync Events & Clients for v2.2 #1127

Merged
merged 4 commits into from
Jul 13, 2022
Merged

Log and Return Failed Sync Events & Clients for v2.2 #1127

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9ccc3f2
Handle events and clients that fail to save during sync
qiarie Jul 12, 2022
5e8f695
Bump up version to 2.10.4-SNAPSHOT
qiarie Jul 12, 2022
1bf1e43
Upgrade server-core dependency to 2.14.4-SNAPSHOT
qiarie Jul 13, 2022
d1cf766
Remove wildcard imports
qiarie Jul 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

<artifactId>opensrp-server-web</artifactId>
<packaging>war</packaging>
<version>2.10.3-SNAPSHOT</version>
<version>2.10.4-SNAPSHOT</version>
<name>opensrp-server-web</name>
<description>OpenSRP Server Web Application</description>
<url>https://github.com/OpenSRP/opensrp-server-web</url>
Expand All @@ -24,7 +24,7 @@
<redis.jedis.version>3.8.0</redis.jedis.version>
<opensrp.updatePolicy>always</opensrp.updatePolicy>
<nexus-staging-maven-plugin.version>1.5.1</nexus-staging-maven-plugin.version>
<opensrp.core.version>2.14.3-SNAPSHOT</opensrp.core.version>
<opensrp.core.version>2.14.4-SNAPSHOT</opensrp.core.version>
<opensrp.connector.version>2.4.1-SNAPSHOT</opensrp.connector.version>
<opensrp.interface.version>2.0.1-SNAPSHOT</opensrp.interface.version>
<powermock.version>2.0.5</powermock.version>
Expand Down
120 changes: 62 additions & 58 deletions src/main/java/org/opensrp/web/rest/EventResource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,51 +1,13 @@
package org.opensrp.web.rest;

import static org.opensrp.common.AllConstants.CLIENTS_FETCH_BATCH_SIZE;
import static org.opensrp.common.AllConstants.BaseEntity.BASE_ENTITY_ID;
import static org.opensrp.common.AllConstants.BaseEntity.LAST_UPDATE;
import static org.opensrp.common.AllConstants.Event.ENTITY_TYPE;
import static org.opensrp.common.AllConstants.Event.EVENT_DATE;
import static org.opensrp.common.AllConstants.Event.EVENT_TYPE;
import static org.opensrp.common.AllConstants.Event.LOCATION_ID;
import static org.opensrp.common.AllConstants.Event.PROVIDER_ID;
import static org.opensrp.common.AllConstants.Event.TEAM;
import static org.opensrp.common.AllConstants.Event.TEAM_ID;
import static org.opensrp.common.AllConstants.Form.SERVER_VERSION;
import static org.opensrp.util.constants.EventConstants.CASE_NUMBER;
import static org.opensrp.util.constants.EventConstants.EVENT_TYPE_CASE_DETAILS;
import static org.opensrp.web.Constants.RETURN_COUNT;
import static org.opensrp.web.Constants.TOTAL_RECORDS;
import static org.opensrp.web.rest.RestUtils.getDateRangeFilter;
import static org.opensrp.web.rest.RestUtils.getIntegerFilter;
import static org.opensrp.web.rest.RestUtils.getStringFilter;
import static org.opensrp.web.rest.RestUtils.writeToZipFile;
import static org.springframework.http.HttpStatus.BAD_REQUEST;
import static org.springframework.http.HttpStatus.CREATED;
import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR;
import static org.springframework.web.bind.annotation.RequestMethod.POST;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.zip.ZipOutputStream;
import java.util.Map;

import javax.servlet.http.HttpServletRequest;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonArray;
import com.google.gson.reflect.TypeToken;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.apache.commons.io.FileUtils;
import com.google.gson.JsonArray;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager;
Expand All @@ -54,9 +16,10 @@
import org.json.JSONObject;
import org.opensrp.api.domain.User;
import org.opensrp.common.AllConstants.BaseEntity;
import org.opensrp.dto.ExportImagesSummary;
import org.opensrp.domain.Multimedia;
import org.opensrp.dto.ExportEventDataSummary;
import org.opensrp.dto.ExportFlagProblemEventImageMetadata;
import org.opensrp.dto.ExportImagesSummary;
import org.opensrp.search.EventSearchBean;
import org.opensrp.service.ClientService;
import org.opensrp.service.EventService;
Expand All @@ -72,7 +35,6 @@
import org.opensrp.web.utils.Utils;
import org.smartregister.domain.Client;
import org.smartregister.domain.Event;
import org.opensrp.domain.Multimedia;
import org.smartregister.utils.DateTimeTypeConverter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
Expand All @@ -85,16 +47,52 @@
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Controller;
import org.springframework.ui.ModelMap;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.GetMapping;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
import javax.servlet.http.HttpServletRequest;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.zip.ZipOutputStream;

import static org.opensrp.common.AllConstants.BaseEntity.BASE_ENTITY_ID;
import static org.opensrp.common.AllConstants.BaseEntity.LAST_UPDATE;
import static org.opensrp.common.AllConstants.CLIENTS_FETCH_BATCH_SIZE;
import static org.opensrp.common.AllConstants.Event.ENTITY_TYPE;
import static org.opensrp.common.AllConstants.Event.EVENT_DATE;
import static org.opensrp.common.AllConstants.Event.EVENT_TYPE;
import static org.opensrp.common.AllConstants.Event.LOCATION_ID;
import static org.opensrp.common.AllConstants.Event.PROVIDER_ID;
import static org.opensrp.common.AllConstants.Event.TEAM;
import static org.opensrp.common.AllConstants.Event.TEAM_ID;
import static org.opensrp.common.AllConstants.Form.SERVER_VERSION;
import static org.opensrp.util.constants.EventConstants.CASE_NUMBER;
import static org.opensrp.util.constants.EventConstants.EVENT_TYPE_CASE_DETAILS;
import static org.opensrp.web.Constants.RETURN_COUNT;
import static org.opensrp.web.Constants.TOTAL_RECORDS;
import static org.opensrp.web.rest.RestUtils.getDateRangeFilter;
import static org.opensrp.web.rest.RestUtils.getIntegerFilter;
import static org.opensrp.web.rest.RestUtils.getStringFilter;
import static org.opensrp.web.rest.RestUtils.writeToZipFile;
import static org.springframework.http.HttpStatus.BAD_REQUEST;
import static org.springframework.http.HttpStatus.CREATED;
import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR;
import static org.springframework.web.bind.annotation.RequestMethod.POST;

@Controller
@RequestMapping(value = "/rest/event")
Expand Down Expand Up @@ -439,6 +437,7 @@ public ResponseEntity<String> save(@RequestBody String data, Authentication auth
List<String> failedClientsIds = new ArrayList<>();
List<String> failedEventIds = new ArrayList<>();
Map<String, Object> response = new HashMap<>();

try {
long timeBeforeSync = System.currentTimeMillis();
JSONObject syncData = new JSONObject(data);
Expand All @@ -448,8 +447,12 @@ public ResponseEntity<String> save(@RequestBody String data, Authentication auth

if (syncData.has("clients")) {
ArrayList<Client> clients = gson.fromJson(Utils.getStringFromJSON(syncData, "clients"),
new TypeToken<ArrayList<Client>>() {}.getType());
new TypeToken<ArrayList<Client>>() {

}.getType());

logger.info("[SYNC_INFO] {} Clients submitted by user {}", clients.size(), username);

long timeBeforeSavingClients = System.currentTimeMillis();
for (Client client : clients) {
try {
Expand All @@ -459,20 +462,22 @@ public ResponseEntity<String> save(@RequestBody String data, Authentication auth
getExecutionTime(timeBeforeSavingClient));
}
catch (Exception e) {
client.getBaseEntityId();
logger.error("Client {} failed to sync", client.getBaseEntityId(), e);
failedClientsIds.add(client.getId());
failedClientsIds.add(client.getBaseEntityId());
}
}
logger.info("[SYNC_INFO] Saved {} clients in {} seconds", clients.size(),
getExecutionTime(timeBeforeSavingClients));

}

if (syncData.has("events")) {
ArrayList<Event> events = gson.fromJson(Utils.getStringFromJSON(syncData, "events"),
new TypeToken<ArrayList<Event>>() {}.getType());
new TypeToken<ArrayList<Event>>() {

}.getType());

logger.info("[SYNC_INFO] {} Events submitted by user {}", events.size(), username);

long timeBeforeSavingEvents = System.currentTimeMillis();
for (Event event : events) {
try {
Expand All @@ -494,9 +499,8 @@ public ResponseEntity<String> save(@RequestBody String data, Authentication auth
}
}
catch (Exception e) {
logger.error("Event of type {} for client {} failed to sync", event.getEventType(),
event.getBaseEntityId(), e);
failedEventIds.add(event.getId());
logger.error("Event of type {} for client {} failed to sync", event.getEventType(), event.getBaseEntityId(), e);
failedEventIds.add(event.getFormSubmissionId());
}
}
logger.info("[SYNC_INFO] Saved {} events in {} seconds", events.size(),
Expand Down