Skip to content

Commit

Permalink
FRI-590 Added logic to move the backup build to its own bucket
Browse files Browse the repository at this point in the history
  • Loading branch information
@QuyenLy87
QuyenLy87 committed May 24, 2023
1 parent 1ae0cf3 commit 8439e91
Show file tree
Hide file tree
Showing 4 changed files with 77 additions and 25 deletions.
15 changes: 15 additions & 0 deletions src/main/java/org/ihtsdo/buildcloud/core/dao/helper/S3PathHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,15 @@ public class S3PathHelper {
@Value("${srs.published.releases.storage.path}")
private String publishedReleasesStoragePath;

@Value("${srs.published.releases.backup.storage.path}")
private String publishedReleasesBackupStoragePath;

@Value("${srs.publish.job.storage.path}")
private String publishJobStoragePath;

@Value("${srs.publish.job.backup.storage.path}")
private String publishJobBackupStoragePath;

@Value("${srs.externally-maintained.storage.path}")
private String externallyMaintainedStoragePath;

Expand Down Expand Up @@ -150,6 +156,10 @@ public String getPublishJobDirectoryPath(final String releaseCenterKey) {
return getReleaseCenterPath(releaseCenterKey, publishJobStoragePath).toString();
}

public String getPublishJobBackupDirectoryPath(final String releaseCenterKey) {
return getReleaseCenterPath(releaseCenterKey, publishJobBackupStoragePath).toString();
}

public String getPublishJobFilePath(final String releaseCenterKey, final String fileName) {
return getReleaseCenterPath(releaseCenterKey, publishJobStoragePath).append(fileName).toString();
}
Expand All @@ -158,6 +168,11 @@ public String getPublishedReleasesDirectoryPath(final String releaseCenterKey) {
return getReleaseCenterPath(releaseCenterKey, publishedReleasesStoragePath).toString();
}

public String getPublishedReleasesBackupDirectoryPath(final String releaseCenterKey) {
return getReleaseCenterPath(releaseCenterKey, publishedReleasesBackupStoragePath).toString();
}


public String getPublishedReleasesFilePath(final String releaseCenterKey, final String fileName) {
return getReleaseCenterPath(releaseCenterKey, publishedReleasesStoragePath).append(fileName).toString();
}
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/org/ihtsdo/buildcloud/core/service/PublishService.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.ihtsdo.buildcloud.core.service;

import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
Expand All @@ -22,7 +23,7 @@ public interface PublishService {

Map<String, String> getPublishedBuildPathMap(String releaseCenterKey, String productKey);

void publishBuild(Build build, boolean publishComponentIds, String env) throws BusinessServiceException;
void publishBuild(Build build, boolean publishComponentIds, String env) throws BusinessServiceException, IOException;

void publishBuildAsync(Build build, boolean publishComponentIds, String env);

Expand Down
79 changes: 55 additions & 24 deletions src/main/java/org/ihtsdo/buildcloud/core/service/PublishServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,12 @@ public class PublishServiceImpl implements PublishService {

private static final Map<String, ProcessingStatus> concurrentPublishingBuildStatus = new ConcurrentHashMap<>();

@Value("${srs.publish.job.useOwnBackupBucket}")
private Boolean useOwnBackupBucket;

@Value("${srs.publish.job.backup.storage.bucketName}")
private String publishJobBackupStorageBucketName;

@Value("${srs.published.releases.storage.path}")
private String publishedReleasesStoragePath;

Expand Down Expand Up @@ -126,24 +132,38 @@ public List<String> getPublishedPackages(final ReleaseCenter releaseCenter) {
@Override
public List<Build> findPublishedBuilds(String releaseCenterKey, String productKey, boolean includeProdPublishedReleases) throws ResourceNotFoundException {
List<Build> builds = new ArrayList<>();
String buildBckUpPath = s3PathHelper.getPublishJobDirectoryPath(releaseCenterKey) + PUBLISHED_BUILD + S3PathHelper.SEPARATOR + productKey + S3PathHelper.SEPARATOR;
findPublishedBuilds(releaseCenterKey, productKey, builds, buildBckUpPath);
if (includeProdPublishedReleases) {
buildBckUpPath = s3PathHelper.getPublishedReleasesDirectoryPath(releaseCenterKey) + PUBLISHED_BUILD + S3PathHelper.SEPARATOR + productKey + S3PathHelper.SEPARATOR;
findPublishedBuilds(releaseCenterKey, productKey, builds, buildBckUpPath);
if (Boolean.TRUE.equals(useOwnBackupBucket)) {
String buildBckUpPath = s3PathHelper.getPublishJobBackupDirectoryPath(releaseCenterKey) + productKey + S3PathHelper.SEPARATOR;
findPublishedBuilds(this.publishJobBackupStorageBucketName, releaseCenterKey, productKey, builds, buildBckUpPath);
if (includeProdPublishedReleases) {
buildBckUpPath = s3PathHelper.getPublishedReleasesBackupDirectoryPath(releaseCenterKey) + productKey + S3PathHelper.SEPARATOR;
findPublishedBuilds(this.publishJobBackupStorageBucketName, releaseCenterKey, productKey, builds, buildBckUpPath);
}
} else {
String buildBckUpPath = s3PathHelper.getPublishJobDirectoryPath(releaseCenterKey) + PUBLISHED_BUILD + S3PathHelper.SEPARATOR + productKey + S3PathHelper.SEPARATOR;
findPublishedBuilds(this.storageBucketName, releaseCenterKey, productKey, builds, buildBckUpPath);
if (includeProdPublishedReleases) {
buildBckUpPath = s3PathHelper.getPublishedReleasesDirectoryPath(releaseCenterKey) + PUBLISHED_BUILD + S3PathHelper.SEPARATOR + productKey + S3PathHelper.SEPARATOR;
findPublishedBuilds(this.storageBucketName, releaseCenterKey, productKey, builds, buildBckUpPath);
}
}

return builds;
}

@Override
public Map<String, String> getPublishedBuildPathMap(String releaseCenterKey, String productKey) {
Map<String, String> buildPathMap = new HashMap<>();
String buildBckUpPath = s3PathHelper.getPublishJobDirectoryPath(releaseCenterKey) + PUBLISHED_BUILD + S3PathHelper.SEPARATOR + productKey + S3PathHelper.SEPARATOR;
findPublishedBuildPathMap(buildPathMap, buildBckUpPath);
buildBckUpPath = s3PathHelper.getPublishedReleasesDirectoryPath(releaseCenterKey) + PUBLISHED_BUILD + S3PathHelper.SEPARATOR + productKey + S3PathHelper.SEPARATOR;
findPublishedBuildPathMap(buildPathMap, buildBckUpPath);

if (Boolean.TRUE.equals(useOwnBackupBucket)) {
String buildBckUpPath = s3PathHelper.getPublishJobBackupDirectoryPath(releaseCenterKey) + productKey + S3PathHelper.SEPARATOR;
findPublishedBuildPathMap(this.publishJobBackupStorageBucketName, buildPathMap, buildBckUpPath);
buildBckUpPath = s3PathHelper.getPublishedReleasesBackupDirectoryPath(releaseCenterKey) + productKey + S3PathHelper.SEPARATOR;
findPublishedBuildPathMap(this.publishJobBackupStorageBucketName, buildPathMap, buildBckUpPath);
} else {
String buildBckUpPath = s3PathHelper.getPublishJobDirectoryPath(releaseCenterKey) + PUBLISHED_BUILD + S3PathHelper.SEPARATOR + productKey + S3PathHelper.SEPARATOR;
findPublishedBuildPathMap(this.storageBucketName, buildPathMap, buildBckUpPath);
buildBckUpPath = s3PathHelper.getPublishedReleasesDirectoryPath(releaseCenterKey) + PUBLISHED_BUILD + S3PathHelper.SEPARATOR + productKey + S3PathHelper.SEPARATOR;
findPublishedBuildPathMap(this.storageBucketName, buildPathMap, buildBckUpPath);
}
return buildPathMap;
}

Expand All @@ -152,8 +172,8 @@ public Map<String, String> getPublishedBuildPathMap(String releaseCenterKey, Str
public void publishBuildAsync(Build build, boolean publishComponentIds, String env) {
try {
this.publishBuild(build, publishComponentIds, env);
} catch (BusinessServiceException e) {
LOGGER.error("Failed to publish the build {}. Error message: ", build.getId(), e.getMessage());
} catch (Exception e) {
LOGGER.error("Failed to publish the build {}. Error message: {}", build.getId(), e.getMessage());
}
}

Expand All @@ -172,7 +192,7 @@ public ProcessingStatus getPublishingBuildStatus(Build build) {
}

@Override
public void publishBuild(final Build build, boolean publishComponentIds, String env) throws BusinessServiceException {
public void publishBuild(final Build build, boolean publishComponentIds, String env) throws BusinessServiceException, IOException {
MDC.put(BuildService.MDC_BUILD_KEY, build.getUniqueId());

ProcessingStatus currentStatus = concurrentPublishingBuildStatus.get(getBuildUniqueKey(build));
Expand Down Expand Up @@ -222,6 +242,7 @@ public void publishBuild(final Build build, boolean publishComponentIds, String
if (srsFileHelper.exists(publishFilePath)) {
String errorMessage = publishFilePath + " has already been published for Release Center " + build.getReleaseCenterKey() + " (" + build.getCreationTime() + ")";
concurrentPublishingBuildStatus.put(getBuildUniqueKey(build), new ProcessingStatus(Status.FAILED.name(), errorMessage));
LOGGER.error(errorMessage);
throw new EntityAlreadyExistsException(errorMessage);
}
try {
Expand Down Expand Up @@ -268,9 +289,9 @@ public void publishBuild(final Build build, boolean publishComponentIds, String
}
concurrentPublishingBuildStatus.put(getBuildUniqueKey(build), new ProcessingStatus(Status.COMPLETED.name(), null));
}
} catch (IOException e) {
} catch (Exception e) {
concurrentPublishingBuildStatus.put(getBuildUniqueKey(build), new ProcessingStatus(Status.FAILED.name(), "Failed to publish build " + build.getUniqueId() + ". Error message: " + e.getMessage()));
throw new BusinessServiceException("Failed to publish build " + build.getUniqueId(), e);
throw e;
} finally {
MDC.remove(BuildService.MDC_BUILD_KEY);
}
Expand Down Expand Up @@ -617,12 +638,22 @@ private void copyBuildToVersionedContentsStore(String releaseFileFullPath, Strin
private void backupPublishedBuild(Build build) {
String originalBuildPath = s3PathHelper.getBuildPath(build).toString();
List<String> buildFiles = srsFileHelper.listFiles(originalBuildPath);
String buildBckUpPath = s3PathHelper.getPublishJobDirectoryPath(build.getReleaseCenterKey())
+ PUBLISHED_BUILD + S3PathHelper.SEPARATOR
+ build.getProductKey() + S3PathHelper.SEPARATOR
+ build.getId() + S3PathHelper.SEPARATOR;
for (String filename : buildFiles) {
srsFileHelper.copyFile(originalBuildPath + filename, buildBckUpPath + filename);
String buildBckUpPath;
if (Boolean.TRUE.equals(useOwnBackupBucket)) {
buildBckUpPath = s3PathHelper.getPublishJobBackupDirectoryPath(build.getReleaseCenterKey())
+ build.getProductKey() + S3PathHelper.SEPARATOR
+ build.getId() + S3PathHelper.SEPARATOR;
for (String filename : buildFiles) {
srsFileHelper.copyFile(originalBuildPath + filename, publishJobBackupStorageBucketName, buildBckUpPath + filename);
}
} else {
buildBckUpPath = s3PathHelper.getPublishJobDirectoryPath(build.getReleaseCenterKey())
+ PUBLISHED_BUILD + S3PathHelper.SEPARATOR
+ build.getProductKey() + S3PathHelper.SEPARATOR
+ build.getId() + S3PathHelper.SEPARATOR;
for (String filename : buildFiles) {
srsFileHelper.copyFile(originalBuildPath + filename, buildBckUpPath + filename);
}
}
LOGGER.info("Build: {} is copied to path: {}", build.getProductKey() + build.getId(), buildBckUpPath);
}
Expand All @@ -631,7 +662,7 @@ private String getBuildUniqueKey(Build build) {
return build.getReleaseCenterKey() + "|" + build.getProductKey() + "|" + build.getId();
}

private void findPublishedBuildPathMap(Map<String, String> buildPathMap, String buildBckUpPath) {
private void findPublishedBuildPathMap(String storageBucketName, Map<String, String> buildPathMap, String buildBckUpPath) {
final ListObjectsRequest listObjectsRequest = new ListObjectsRequest(storageBucketName, buildBckUpPath, null, null, 10000);
ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);
boolean firstPass = true;
Expand All @@ -654,7 +685,7 @@ private void findPublishedBuildPathMap(Map<String, String> buildPathMap, String
}
}

private void findPublishedBuilds(String releaseCenterKey, String productKey, List<Build> builds, String buildBckUpPath) {
private void findPublishedBuilds(String storageBucketName, String releaseCenterKey, String productKey, List<Build> builds, String buildBckUpPath) {
List<Build> foundBuilds = new ArrayList<>();
final ListObjectsRequest listObjectsRequest = new ListObjectsRequest(storageBucketName, buildBckUpPath, null, null, 10000);
ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);
Expand Down
5 changes: 5 additions & 0 deletions src/main/resources/application.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,11 @@ srs.externally-maintained.storage.path = local/externally-maintained/
srs.build.versioned-content.bucketName = local.snomed.international.bucket
srs.build.versioned-content.path = authoring/version-content/

srs.publish.job.useOwnBackupBucket = false
srs.publish.job.backup.storage.bucketName = local.snomed.published.job.backup
srs.publish.job.backup.storage.path = local
srs.published.releases.backup.storage.path = local

# daily build storage for browser import
dailybuild.storage.readonly = false
dailybuild.storage.local.path = store/local/
Expand Down

0 comments on commit 8439e91

Please sign in to comment.