PIP-231 Clean up any existing data in transformed folder from previou…

…s run during retry
IHTSDO · May 11, 2023 · 82b0071 · 82b0071
1 parent 56ae11b
commit 82b0071
Show file tree

Hide file tree

Showing 5 changed files with 23 additions and 3 deletions.
diff --git a/src/main/java/org/ihtsdo/buildcloud/core/dao/BuildDAO.java b/src/main/java/org/ihtsdo/buildcloud/core/dao/BuildDAO.java
@@ -108,6 +108,8 @@ public interface BuildDAO {
 
 	void deleteOutputFiles(Build build);
 
+	void deleteTransformedFiles(Build build);
+
 	InputStream getBuildInputGatherReportStream(Build build);
 
 	boolean isDerivativeProduct(Build build);

diff --git a/src/main/java/org/ihtsdo/buildcloud/core/dao/BuildDAOImpl.java b/src/main/java/org/ihtsdo/buildcloud/core/dao/BuildDAOImpl.java
@@ -1043,9 +1043,21 @@ public boolean isBuildCancelRequested(final Build build) {
 	@Override
 	public void deleteOutputFiles(Build build) {
 		List<String> outputFiles = listOutputFilePaths(build);
+		final String outputFilesPath = pathHelper.getOutputFilesPath(build);
 		for (String outputFile : outputFiles) {
-			if (srsFileHelper.exists(outputFile)) {
-				srsFileHelper.deleteFile(outputFile);
+			if (srsFileHelper.exists(outputFilesPath + outputFile)) {
+				srsFileHelper.deleteFile(outputFilesPath + outputFile);
+			}
+		}
+	}
+
+	@Override
+	public void deleteTransformedFiles(Build build) {
+		List<String> transformedFiles = listTransformedFilePaths(build);
+		final String transformedFilesPath = pathHelper.getBuildTransformedFilesPath(build).toString();
+		for (String transformedFile : transformedFiles) {
+			if (srsFileHelper.exists(transformedFilesPath + transformedFile)) {
+				srsFileHelper.deleteFile(transformedFilesPath + transformedFile);
 			}
 		}
 	}

diff --git a/src/main/java/org/ihtsdo/buildcloud/core/dao/helper/S3PathHelper.java b/src/main/java/org/ihtsdo/buildcloud/core/dao/helper/S3PathHelper.java
@@ -139,7 +139,7 @@ private String getFilePath(final Build build, final String relativePath) {
 	}
 
 	public StringBuilder getBuildTransformedFilesPath(final Build build) {
-		return getBuildPath(build.getReleaseCenterKey(), build.getProductKey(), build.getId()).append(TRANSFORMED_FILES).append(SEPARATOR);
+		return getBuildPath(build).append(TRANSFORMED_FILES).append(SEPARATOR);
 	}
 
 	public String getTransformedFilePath(final Build build, final String relativeFilePath) {

diff --git a/src/main/java/org/ihtsdo/buildcloud/core/service/build/Rf2FileExportRunner.java b/src/main/java/org/ihtsdo/buildcloud/core/service/build/Rf2FileExportRunner.java
@@ -48,6 +48,9 @@ public Rf2FileExportRunner(final Build build, final BuildDAO dao, final int maxR
 	}
 
 	public final void generateReleaseFiles() throws ReleaseFileGenerationException {
+		//  clean up any existing data in output folder in case there could be data left from previous run during retry
+		buildDao.deleteOutputFiles(build);
+
 		final List<String> transformedFiles = getTransformedDeltaFiles();
 		final Set<String> newRF2InputFiles = configuration.getNewRF2InputFileSet();
 		final Map<String,Set<String>> includedFilesMap = configuration.getIncludedFilesInNewFilesMap();

diff --git a/src/main/java/org/ihtsdo/buildcloud/core/service/build/transform/TransformationService.java b/src/main/java/org/ihtsdo/buildcloud/core/service/build/transform/TransformationService.java
@@ -89,6 +89,9 @@ public TransformationService() {
 	public void transformFiles(final Build build, final Map<String, TableSchema> inputFileSchemaMap)
 			throws BusinessServiceException, NoSuchAlgorithmException {
 
+		//  clean up any existing data in transformed folder in case there could be data left from previous run during retry
+		dao.deleteTransformedFiles(build);
+
 		BuildConfiguration configuration = build.getConfiguration();
 		final BuildReport report = build.getBuildReport();