diff --git a/src/main/java/org/ihtsdo/buildcloud/core/dao/helper/S3PathHelper.java b/src/main/java/org/ihtsdo/buildcloud/core/dao/helper/S3PathHelper.java index f092c5f3e..1fbae4ffb 100644 --- a/src/main/java/org/ihtsdo/buildcloud/core/dao/helper/S3PathHelper.java +++ b/src/main/java/org/ihtsdo/buildcloud/core/dao/helper/S3PathHelper.java @@ -19,9 +19,15 @@ public class S3PathHelper { @Value("${srs.published.releases.storage.path}") private String publishedReleasesStoragePath; + @Value("${srs.published.releases.backup.storage.path}") + private String publishedReleasesBackupStoragePath; + @Value("${srs.publish.job.storage.path}") private String publishJobStoragePath; + @Value("${srs.publish.job.backup.storage.path}") + private String publishJobBackupStoragePath; + @Value("${srs.externally-maintained.storage.path}") private String externallyMaintainedStoragePath; @@ -150,6 +156,10 @@ public String getPublishJobDirectoryPath(final String releaseCenterKey) { return getReleaseCenterPath(releaseCenterKey, publishJobStoragePath).toString(); } + public String getPublishJobBackupDirectoryPath(final String releaseCenterKey) { + return getReleaseCenterPath(releaseCenterKey, publishJobBackupStoragePath).toString(); + } + public String getPublishJobFilePath(final String releaseCenterKey, final String fileName) { return getReleaseCenterPath(releaseCenterKey, publishJobStoragePath).append(fileName).toString(); } @@ -158,6 +168,11 @@ public String getPublishedReleasesDirectoryPath(final String releaseCenterKey) { return getReleaseCenterPath(releaseCenterKey, publishedReleasesStoragePath).toString(); } + public String getPublishedReleasesBackupDirectoryPath(final String releaseCenterKey) { + return getReleaseCenterPath(releaseCenterKey, publishedReleasesBackupStoragePath).toString(); + } + + public String getPublishedReleasesFilePath(final String releaseCenterKey, final String fileName) { return getReleaseCenterPath(releaseCenterKey, publishedReleasesStoragePath).append(fileName).toString(); } diff --git a/src/main/java/org/ihtsdo/buildcloud/core/service/PublishService.java b/src/main/java/org/ihtsdo/buildcloud/core/service/PublishService.java index 078ec5fcc..2828774db 100644 --- a/src/main/java/org/ihtsdo/buildcloud/core/service/PublishService.java +++ b/src/main/java/org/ihtsdo/buildcloud/core/service/PublishService.java @@ -1,5 +1,6 @@ package org.ihtsdo.buildcloud.core.service; +import java.io.IOException; import java.io.InputStream; import java.util.List; import java.util.Map; @@ -22,7 +23,7 @@ public interface PublishService { Map getPublishedBuildPathMap(String releaseCenterKey, String productKey); - void publishBuild(Build build, boolean publishComponentIds, String env) throws BusinessServiceException; + void publishBuild(Build build, boolean publishComponentIds, String env) throws BusinessServiceException, IOException; void publishBuildAsync(Build build, boolean publishComponentIds, String env); diff --git a/src/main/java/org/ihtsdo/buildcloud/core/service/PublishServiceImpl.java b/src/main/java/org/ihtsdo/buildcloud/core/service/PublishServiceImpl.java index 86e485a2b..d7839a12f 100644 --- a/src/main/java/org/ihtsdo/buildcloud/core/service/PublishServiceImpl.java +++ b/src/main/java/org/ihtsdo/buildcloud/core/service/PublishServiceImpl.java @@ -63,6 +63,12 @@ public class PublishServiceImpl implements PublishService { private static final Map concurrentPublishingBuildStatus = new ConcurrentHashMap<>(); + @Value("${srs.publish.job.useOwnBackupBucket}") + private Boolean useOwnBackupBucket; + + @Value("${srs.publish.job.backup.storage.bucketName}") + private String publishJobBackupStorageBucketName; + @Value("${srs.published.releases.storage.path}") private String publishedReleasesStoragePath; @@ -126,24 +132,38 @@ public List getPublishedPackages(final ReleaseCenter releaseCenter) { @Override public List findPublishedBuilds(String releaseCenterKey, String productKey, boolean includeProdPublishedReleases) throws ResourceNotFoundException { List builds = new ArrayList<>(); - String buildBckUpPath = s3PathHelper.getPublishJobDirectoryPath(releaseCenterKey) + PUBLISHED_BUILD + S3PathHelper.SEPARATOR + productKey + S3PathHelper.SEPARATOR; - findPublishedBuilds(releaseCenterKey, productKey, builds, buildBckUpPath); - if (includeProdPublishedReleases) { - buildBckUpPath = s3PathHelper.getPublishedReleasesDirectoryPath(releaseCenterKey) + PUBLISHED_BUILD + S3PathHelper.SEPARATOR + productKey + S3PathHelper.SEPARATOR; - findPublishedBuilds(releaseCenterKey, productKey, builds, buildBckUpPath); + if (Boolean.TRUE.equals(useOwnBackupBucket)) { + String buildBckUpPath = s3PathHelper.getPublishJobBackupDirectoryPath(releaseCenterKey) + productKey + S3PathHelper.SEPARATOR; + findPublishedBuilds(this.publishJobBackupStorageBucketName, releaseCenterKey, productKey, builds, buildBckUpPath); + if (includeProdPublishedReleases) { + buildBckUpPath = s3PathHelper.getPublishedReleasesBackupDirectoryPath(releaseCenterKey) + productKey + S3PathHelper.SEPARATOR; + findPublishedBuilds(this.publishJobBackupStorageBucketName, releaseCenterKey, productKey, builds, buildBckUpPath); + } + } else { + String buildBckUpPath = s3PathHelper.getPublishJobDirectoryPath(releaseCenterKey) + PUBLISHED_BUILD + S3PathHelper.SEPARATOR + productKey + S3PathHelper.SEPARATOR; + findPublishedBuilds(this.storageBucketName, releaseCenterKey, productKey, builds, buildBckUpPath); + if (includeProdPublishedReleases) { + buildBckUpPath = s3PathHelper.getPublishedReleasesDirectoryPath(releaseCenterKey) + PUBLISHED_BUILD + S3PathHelper.SEPARATOR + productKey + S3PathHelper.SEPARATOR; + findPublishedBuilds(this.storageBucketName, releaseCenterKey, productKey, builds, buildBckUpPath); + } } - return builds; } @Override public Map getPublishedBuildPathMap(String releaseCenterKey, String productKey) { Map buildPathMap = new HashMap<>(); - String buildBckUpPath = s3PathHelper.getPublishJobDirectoryPath(releaseCenterKey) + PUBLISHED_BUILD + S3PathHelper.SEPARATOR + productKey + S3PathHelper.SEPARATOR; - findPublishedBuildPathMap(buildPathMap, buildBckUpPath); - buildBckUpPath = s3PathHelper.getPublishedReleasesDirectoryPath(releaseCenterKey) + PUBLISHED_BUILD + S3PathHelper.SEPARATOR + productKey + S3PathHelper.SEPARATOR; - findPublishedBuildPathMap(buildPathMap, buildBckUpPath); - + if (Boolean.TRUE.equals(useOwnBackupBucket)) { + String buildBckUpPath = s3PathHelper.getPublishJobBackupDirectoryPath(releaseCenterKey) + productKey + S3PathHelper.SEPARATOR; + findPublishedBuildPathMap(this.publishJobBackupStorageBucketName, buildPathMap, buildBckUpPath); + buildBckUpPath = s3PathHelper.getPublishedReleasesBackupDirectoryPath(releaseCenterKey) + productKey + S3PathHelper.SEPARATOR; + findPublishedBuildPathMap(this.publishJobBackupStorageBucketName, buildPathMap, buildBckUpPath); + } else { + String buildBckUpPath = s3PathHelper.getPublishJobDirectoryPath(releaseCenterKey) + PUBLISHED_BUILD + S3PathHelper.SEPARATOR + productKey + S3PathHelper.SEPARATOR; + findPublishedBuildPathMap(this.storageBucketName, buildPathMap, buildBckUpPath); + buildBckUpPath = s3PathHelper.getPublishedReleasesDirectoryPath(releaseCenterKey) + PUBLISHED_BUILD + S3PathHelper.SEPARATOR + productKey + S3PathHelper.SEPARATOR; + findPublishedBuildPathMap(this.storageBucketName, buildPathMap, buildBckUpPath); + } return buildPathMap; } @@ -152,8 +172,8 @@ public Map getPublishedBuildPathMap(String releaseCenterKey, Str public void publishBuildAsync(Build build, boolean publishComponentIds, String env) { try { this.publishBuild(build, publishComponentIds, env); - } catch (BusinessServiceException e) { - LOGGER.error("Failed to publish the build {}. Error message: ", build.getId(), e.getMessage()); + } catch (Exception e) { + LOGGER.error("Failed to publish the build {}. Error message: {}", build.getId(), e.getMessage()); } } @@ -172,7 +192,7 @@ public ProcessingStatus getPublishingBuildStatus(Build build) { } @Override - public void publishBuild(final Build build, boolean publishComponentIds, String env) throws BusinessServiceException { + public void publishBuild(final Build build, boolean publishComponentIds, String env) throws BusinessServiceException, IOException { MDC.put(BuildService.MDC_BUILD_KEY, build.getUniqueId()); ProcessingStatus currentStatus = concurrentPublishingBuildStatus.get(getBuildUniqueKey(build)); @@ -222,6 +242,7 @@ public void publishBuild(final Build build, boolean publishComponentIds, String if (srsFileHelper.exists(publishFilePath)) { String errorMessage = publishFilePath + " has already been published for Release Center " + build.getReleaseCenterKey() + " (" + build.getCreationTime() + ")"; concurrentPublishingBuildStatus.put(getBuildUniqueKey(build), new ProcessingStatus(Status.FAILED.name(), errorMessage)); + LOGGER.error(errorMessage); throw new EntityAlreadyExistsException(errorMessage); } try { @@ -268,9 +289,9 @@ public void publishBuild(final Build build, boolean publishComponentIds, String } concurrentPublishingBuildStatus.put(getBuildUniqueKey(build), new ProcessingStatus(Status.COMPLETED.name(), null)); } - } catch (IOException e) { + } catch (Exception e) { concurrentPublishingBuildStatus.put(getBuildUniqueKey(build), new ProcessingStatus(Status.FAILED.name(), "Failed to publish build " + build.getUniqueId() + ". Error message: " + e.getMessage())); - throw new BusinessServiceException("Failed to publish build " + build.getUniqueId(), e); + throw e; } finally { MDC.remove(BuildService.MDC_BUILD_KEY); } @@ -617,12 +638,22 @@ private void copyBuildToVersionedContentsStore(String releaseFileFullPath, Strin private void backupPublishedBuild(Build build) { String originalBuildPath = s3PathHelper.getBuildPath(build).toString(); List buildFiles = srsFileHelper.listFiles(originalBuildPath); - String buildBckUpPath = s3PathHelper.getPublishJobDirectoryPath(build.getReleaseCenterKey()) - + PUBLISHED_BUILD + S3PathHelper.SEPARATOR - + build.getProductKey() + S3PathHelper.SEPARATOR - + build.getId() + S3PathHelper.SEPARATOR; - for (String filename : buildFiles) { - srsFileHelper.copyFile(originalBuildPath + filename, buildBckUpPath + filename); + String buildBckUpPath; + if (Boolean.TRUE.equals(useOwnBackupBucket)) { + buildBckUpPath = s3PathHelper.getPublishJobBackupDirectoryPath(build.getReleaseCenterKey()) + + build.getProductKey() + S3PathHelper.SEPARATOR + + build.getId() + S3PathHelper.SEPARATOR; + for (String filename : buildFiles) { + srsFileHelper.copyFile(originalBuildPath + filename, publishJobBackupStorageBucketName, buildBckUpPath + filename); + } + } else { + buildBckUpPath = s3PathHelper.getPublishJobDirectoryPath(build.getReleaseCenterKey()) + + PUBLISHED_BUILD + S3PathHelper.SEPARATOR + + build.getProductKey() + S3PathHelper.SEPARATOR + + build.getId() + S3PathHelper.SEPARATOR; + for (String filename : buildFiles) { + srsFileHelper.copyFile(originalBuildPath + filename, buildBckUpPath + filename); + } } LOGGER.info("Build: {} is copied to path: {}", build.getProductKey() + build.getId(), buildBckUpPath); } @@ -631,7 +662,7 @@ private String getBuildUniqueKey(Build build) { return build.getReleaseCenterKey() + "|" + build.getProductKey() + "|" + build.getId(); } - private void findPublishedBuildPathMap(Map buildPathMap, String buildBckUpPath) { + private void findPublishedBuildPathMap(String storageBucketName, Map buildPathMap, String buildBckUpPath) { final ListObjectsRequest listObjectsRequest = new ListObjectsRequest(storageBucketName, buildBckUpPath, null, null, 10000); ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); boolean firstPass = true; @@ -654,7 +685,7 @@ private void findPublishedBuildPathMap(Map buildPathMap, String } } - private void findPublishedBuilds(String releaseCenterKey, String productKey, List builds, String buildBckUpPath) { + private void findPublishedBuilds(String storageBucketName, String releaseCenterKey, String productKey, List builds, String buildBckUpPath) { List foundBuilds = new ArrayList<>(); final ListObjectsRequest listObjectsRequest = new ListObjectsRequest(storageBucketName, buildBckUpPath, null, null, 10000); ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index c41016098..c40f2c8f2 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -93,6 +93,11 @@ srs.externally-maintained.storage.path = local/externally-maintained/ srs.build.versioned-content.bucketName = local.snomed.international.bucket srs.build.versioned-content.path = authoring/version-content/ +srs.publish.job.useOwnBackupBucket = false +srs.publish.job.backup.storage.bucketName = local.snomed.published.job.backup +srs.publish.job.backup.storage.path = local +srs.published.releases.backup.storage.path = local + # daily build storage for browser import dailybuild.storage.readonly = false dailybuild.storage.local.path = store/local/