Skip to content

Commit

Permalink
initial commit
Browse files Browse the repository at this point in the history
  • Loading branch information
@QuyenLy87
QuyenLy87 committed Oct 7, 2022
1 parent d1ff286 commit b2283b3
Show file tree
Hide file tree
Showing 26 changed files with 481 additions and 34 deletions.
14 changes: 13 additions & 1 deletion src/main/java/org/ihtsdo/buildcloud/core/entity/BuildConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,13 @@ public class BuildConfiguration {

@Column(name="previous_published_release")
private String previousPublishedPackage;

@Column(name="additional_previous_published_releases")
private String additionalPreviousPublishedPackages;

@Column(name="rf2_input_files")
private String newRF2InputFiles;

@Type(type="yes_no")
@Column(name="just_package")
private boolean justPackage = false;
Expand Down Expand Up @@ -284,6 +288,14 @@ public void setPreviousPublishedPackage(final String previousPublishedPackage) {
this.previousPublishedPackage = previousPublishedPackage;
}

public String getAdditionalPreviousPublishedPackages() {
return additionalPreviousPublishedPackages;
}

public void setAdditionalPreviousPublishedPackages(String additionalPreviousPublishedPackages) {
this.additionalPreviousPublishedPackages = additionalPreviousPublishedPackages;
}

public boolean isCreateLegacyIds() {
return createLegacyIds;
}
Expand Down
6 changes: 4 additions & 2 deletions src/main/java/org/ihtsdo/buildcloud/core/service/InputFileServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,11 @@ public class InputFileServiceImpl implements InputFileService {

private final FileHelper fileHelper;

private static final String SRC_TERM_SERVER = "terminology-server";
public static final String SRC_TERM_SERVER = "terminology-server";

private static final String SRC_EXT_MAINTAINED = "externally-maintained";
public static final String SRC_EXT_MAINTAINED = "externally-maintained";

public static final String SRC_EXTERNAL_DEPENDENCY = "external-dependency-package";

@Autowired
private InputFileDAO inputFileDAO;
Expand Down
1 change: 1 addition & 0 deletions src/main/java/org/ihtsdo/buildcloud/core/service/ProductService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ public interface ProductService extends EntityService<Product> {
String BETA_RELEASE = "betaRelease";
String DAILY_BUILD = "dailyBuild";
String PREVIOUS_PUBLISHED_PACKAGE = "previousPublishedPackage";
String ADDITIONAL_PREVIOUS_PUBLISHED_PACKAGES = "additionalPreviousPublishedPackages";
String README_END_DATE = "readmeEndDate";
String WORKBENCH_DATA_FIXES_REQUIRED = "workbenchDataFixesRequired";
String INPUT_FILES_FIXES_REQUIRED = "inputFilesFixesRequired";
Expand Down
17 changes: 17 additions & 0 deletions src/main/java/org/ihtsdo/buildcloud/core/service/ProductServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ public class ProductServiceImpl extends EntityServiceImpl<Product> implements Pr

private static final Logger LOGGER = LoggerFactory.getLogger(ProductServiceImpl.class);

public static final String EXTERNAL_DEPENDENCY_PACKAGE_RELEASE_CENTER = "external_dependency_packages";

@Autowired
private ProductDAO productDAO;

Expand Down Expand Up @@ -361,6 +363,21 @@ private void updateProductBuildConfiguration(final Map<String, String> newProper
}
}

if (newPropertyValues.containsKey(ADDITIONAL_PREVIOUS_PUBLISHED_PACKAGES)) {
final String additionalPackagesStr = newPropertyValues.get(ADDITIONAL_PREVIOUS_PUBLISHED_PACKAGES);
if (StringUtils.isEmpty(additionalPackagesStr)) {
configuration.setAdditionalPreviousPublishedPackages(null);
} else {
String[] additionalPackageArr = Arrays.stream(additionalPackagesStr.split(",")).map(String::trim).toArray(String[]::new);
for (String previousPublishedPackage : additionalPackageArr) {
if (!publishService.exists(EXTERNAL_DEPENDENCY_PACKAGE_RELEASE_CENTER, previousPublishedPackage)) {
throw new ResourceNotFoundException("Could not find the additional previously published package: " + previousPublishedPackage);
}
}
configuration.setAdditionalPreviousPublishedPackages(additionalPackagesStr);
}
}

if (newPropertyValues.containsKey(CUSTOM_REFSET_COMPOSITE_KEYS)) {
final Map<String, List<Integer>> refsetCompositeKeyMap = new HashMap<>();
try {
Expand Down
2 changes: 2 additions & 0 deletions src/main/java/org/ihtsdo/buildcloud/core/service/PublishService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,4 +31,6 @@ public interface PublishService {
void publishAdHocFile(ReleaseCenter releaseCenter, InputStream inputStream, String originalFilename, long size, boolean publishComponentIds) throws BusinessServiceException;

boolean exists(ReleaseCenter releaseCenter, String previouslyPublishedPackageName);

boolean exists(String releaseCenterKey, String previouslyPublishedPackageName);
}
9 changes: 7 additions & 2 deletions src/main/java/org/ihtsdo/buildcloud/core/service/PublishServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -333,12 +333,17 @@ public void publishAdHocFile(ReleaseCenter releaseCenter, InputStream inputStrea
// For scenarios in UAT and DEV where we use locally published release packages for a new build,
// if the file is not found in ${srs.published.releases.storage.path}, then look in ${srs.publish.job.storage.path}
public boolean exists(final ReleaseCenter releaseCenter, final String targetFileName) {
String path = s3PathHelper.getPublishedReleasesFilePath(releaseCenter.getBusinessKey(), targetFileName);
return exists(releaseCenter.getBusinessKey(), targetFileName);
}

@Override
public boolean exists(String releaseCenterKey, String targetFileName) {
String path = s3PathHelper.getPublishedReleasesFilePath(releaseCenterKey, targetFileName);
LOGGER.info("Check if published file exists for path {} in storage bucket", path);
boolean exists = srsFileHelper.exists(path);

if (!exists && !publishedReleasesStoragePath.equals(publishJobStoragePath)) {
path = s3PathHelper.getPublishJobFilePath(releaseCenter.getBusinessKey(), targetFileName);
path = s3PathHelper.getPublishJobFilePath(releaseCenterKey, targetFileName);
LOGGER.info("Check if published file exists for path {} in storage bucket", path);
exists = srsFileHelper.exists(path);
}
Expand Down
191 changes: 178 additions & 13 deletions src/main/java/org/ihtsdo/buildcloud/core/service/build/Rf2FileExportRunner.java
View file
Original file line number Diff line number Diff line change
@@ -1,34 +1,40 @@
package org.ihtsdo.buildcloud.core.service.build;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.time.Duration;
import java.time.Instant;
import java.util.*;

import com.amazonaws.AmazonClientException;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.commons.lang3.time.FastDateFormat;
import org.ihtsdo.buildcloud.core.dao.BuildDAO;
import org.ihtsdo.buildcloud.core.dao.io.AsyncPipedStreamBean;
import org.ihtsdo.buildcloud.core.entity.Build;
import org.ihtsdo.buildcloud.core.entity.BuildConfiguration;
import org.ihtsdo.buildcloud.core.entity.ExtensionConfig;
import org.ihtsdo.buildcloud.core.entity.ReleaseCenter;
import org.ihtsdo.buildcloud.core.manifest.FolderType;
import org.ihtsdo.buildcloud.core.manifest.ListingType;
import org.ihtsdo.buildcloud.core.service.build.database.RF2TableExportDAO;
import org.ihtsdo.buildcloud.core.service.build.database.RF2TableResults;
import org.ihtsdo.buildcloud.core.service.build.database.Rf2FileWriter;
import org.ihtsdo.buildcloud.core.service.build.database.map.RF2TableExportDAOImpl;
import org.ihtsdo.buildcloud.core.service.helper.ManifestXmlFileParser;
import org.ihtsdo.buildcloud.core.service.helper.StatTimer;
import org.ihtsdo.buildcloud.core.service.validation.precondition.ManifestFileListingHelper;
import org.ihtsdo.otf.rest.exception.BusinessServiceException;
import org.ihtsdo.otf.rest.exception.ResourceNotFoundException;
import org.ihtsdo.snomed.util.rf2.schema.ComponentType;
import org.ihtsdo.snomed.util.rf2.schema.TableSchema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.amazonaws.AmazonClientException;
import javax.xml.bind.JAXBException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.time.Duration;
import java.time.Instant;
import java.util.*;

import static org.ihtsdo.buildcloud.core.entity.BuildConfiguration.BETA_PREFIX;
import static org.ihtsdo.buildcloud.core.service.InputFileServiceImpl.SRC_EXTERNAL_DEPENDENCY;
import static org.ihtsdo.buildcloud.core.service.ProductServiceImpl.EXTERNAL_DEPENDENCY_PACKAGE_RELEASE_CENTER;
import static org.ihtsdo.buildcloud.core.service.build.RF2Constants.*;

public class Rf2FileExportRunner {
Expand Down Expand Up @@ -59,21 +65,78 @@ public final void generateReleaseFiles() throws ReleaseFileGenerationException {
boolean success = false;
while (!success) {
try {
boolean fileFirstTimeRelease = false;
boolean fileFirstTimeRelease;
String cleanFileName = thisFile;
if (configuration.isBetaRelease()) {
cleanFileName = thisFile.substring(1);
}
final boolean newFile = newRF2InputFiles.contains(cleanFileName.replace(RF2Constants.SCT2, RF2Constants.INPUT_FILE_PREFIX).replace(RF2Constants.DER2, RF2Constants.INPUT_FILE_PREFIX));
cleanFileName = cleanFileName.replace(RF2Constants.SCT2, RF2Constants.INPUT_FILE_PREFIX).replace(RF2Constants.DER2, RF2Constants.INPUT_FILE_PREFIX);
final boolean newFile = newRF2InputFiles.contains(cleanFileName);
fileFirstTimeRelease = newFile || configuration.isFirstTimeRelease();
Set<String> includedFilesInNewFile = includedFilesMap.get(cleanFileName.replace(RF2Constants.SCT2, RF2Constants.INPUT_FILE_PREFIX).replace(RF2Constants.DER2, RF2Constants.INPUT_FILE_PREFIX));
Set<String> includedFilesInNewFile = includedFilesMap.get(cleanFileName);
generateReleaseFile(thisFile, configuration.getCustomRefsetCompositeKeys(), fileFirstTimeRelease, includedFilesInNewFile);
success = true;
} catch (final Exception e) {
failureCount = handleException(e, thisFile, failureCount);
}
}
}

// Copy files from the additional previous published releases
final List<String> filesFromAdditionalPackages = getAdditionalFilesFromManifest();
for ( String thisFile : filesFromAdditionalPackages) {
if (!thisFile.endsWith(RF2Constants.TXT_FILE_EXTENSION)) {
continue;
}
int failureCount = 0;
boolean success = false;
while (!success) {
try {
generateAdditionalReleaseFile(thisFile, configuration.getCustomRefsetCompositeKeys());
success = true;
} catch (final Exception e) {
failureCount = handleException(e, thisFile, failureCount);
}
}
}
}

private List<String> getAdditionalFilesFromManifest() {
try (InputStream manifestInputSteam = buildDao.getManifestStream(build)) {
final ManifestXmlFileParser parser = new ManifestXmlFileParser();
final ListingType listingType = parser.parse(manifestInputSteam);
FolderType folderType = listingType.getFolder();
List<FolderType> folderTypes = folderType.getFolder();
boolean isDeltaFolderExistInManifest = false;
for (FolderType subFolderType : folderTypes) {
if (subFolderType.getName().equalsIgnoreCase(RF2Constants.DELTA)) {
isDeltaFolderExistInManifest = true;
break;
}
}
for (FolderType subFolderTypeLevel1 : folderTypes) {
if ((isDeltaFolderExistInManifest && subFolderTypeLevel1.getName().equalsIgnoreCase(RF2Constants.DELTA))
|| (!isDeltaFolderExistInManifest && subFolderTypeLevel1.getName().equalsIgnoreCase(RF2Constants.SNAPSHOT)) ) {
List<String> files = new ArrayList <>();
for (FolderType subFolderTypeLevel2 : subFolderTypeLevel1.getFolder()) {
files = ManifestFileListingHelper.listAllFiles(subFolderTypeLevel2, SRC_EXTERNAL_DEPENDENCY);
}
// Rename Snapshot to Delta
if (files.size() > 0 && !isDeltaFolderExistInManifest) {
List<String> newFiles = new ArrayList<>();
for (String file : files) {
newFiles.add(file.replace("Snapshot_", "Delta_").replace("Snapshot-", "Delta-"));
}
return newFiles;
}
return files;
}
}
} catch (ResourceNotFoundException | JAXBException | IOException e) {
LOGGER.error("Failed to parse manifest xml file." + e.getMessage());
}

return Collections.emptyList();
}

public boolean isInferredRelationshipFileExist(final List<String> rf2DeltaFilesSpecifiedByManifest) throws ReleaseFileGenerationException{
Expand Down Expand Up @@ -259,6 +322,97 @@ private void generateReleaseFile(final String transformedDeltaDataFile, final Ma
}
}


private void generateAdditionalReleaseFile(final String additionalDeltaDataFile, final Map<String, List <Integer>> customRefsetCompositeKeys) throws BusinessServiceException {

if (StringUtils.isEmpty(configuration.getAdditionalPreviousPublishedPackages())) {
throw new BusinessServiceException("The additional previous published packages must not be empty");
}

String[] additionalPreviousPublishedPackages = Arrays.stream(configuration.getAdditionalPreviousPublishedPackages().split(",")).map(String::trim).toArray(String[]::new);

LOGGER.info("Generating additional release file using {}", additionalDeltaDataFile);
final StatTimer timer = new StatTimer(getClass());
final Rf2FileWriter rf2FileWriter = new Rf2FileWriter(configuration.getExcludeRefsetDescriptorMembers(), configuration.getExcludeLanguageRefsetIds());
RF2TableExportDAO rf2TableDAO = null;
TableSchema tableSchema = null;
try {

LOGGER.debug("Start: creating table for {}", additionalDeltaDataFile);
rf2TableDAO = new RF2TableExportDAOImpl(customRefsetCompositeKeys);
timer.split();

for (String additionalPreviousPublishedPackage : additionalPreviousPublishedPackages) {
InputStream additionalDeltaInputStream = getEquivalentAdditionalPackageInputStream(additionalPreviousPublishedPackage, additionalDeltaDataFile);
if (additionalDeltaInputStream != null) {
tableSchema = rf2TableDAO.createTable(additionalDeltaDataFile, additionalDeltaInputStream, false);
LOGGER.debug("Start: Exporting delta file for {}", tableSchema.getTableName());
timer.setTargetEntity(tableSchema.getTableName());
timer.logTimeTaken("Create table");

// Export ordered Delta file
final AsyncPipedStreamBean deltaFileAsyncPipe = buildDao.getOutputFileOutputStream(build, additionalDeltaDataFile);

timer.split();
RF2TableResults deltaResultSet = rf2TableDAO.selectAllOrdered(tableSchema);
timer.logTimeTaken("Select all ordered");
timer.split();
rf2FileWriter.exportDelta(deltaResultSet, tableSchema, deltaFileAsyncPipe.getOutputStream());
LOGGER.debug("Completed processing delta file for {}, waiting for network", tableSchema.getTableName());
timer.logTimeTaken("Export delta processing");
deltaFileAsyncPipe.waitForFinish();
LOGGER.debug("Finish: Exporting delta file for {}", tableSchema.getTableName());
break;
}
}

final String currentFullFileName = constructFullOrSnapshotFilename(additionalDeltaDataFile, RF2Constants.FULL);
final String snapshotOutputFilePath = constructFullOrSnapshotFilename(additionalDeltaDataFile, SNAPSHOT);
for (String additionalPreviousPublishedPackage : additionalPreviousPublishedPackages) {
InputStream additionalFullInputStream = getEquivalentAdditionalPackageInputStream(additionalPreviousPublishedPackage, currentFullFileName);
if (additionalFullInputStream != null) {
tableSchema = rf2TableDAO.createTable(additionalDeltaDataFile, additionalFullInputStream, false);
LOGGER.debug("Start: Exporting Full file for {}", tableSchema.getTableName());
timer.setTargetEntity(tableSchema.getTableName());
timer.logTimeTaken("Create table");

// Export Full and Snapshot files
final AsyncPipedStreamBean fullFileAsyncPipe = buildDao.getOutputFileOutputStream(build, currentFullFileName);
final AsyncPipedStreamBean snapshotAsyncPipe = buildDao.getOutputFileOutputStream(build, snapshotOutputFilePath);

timer.split();
final RF2TableResults fullResultSet = rf2TableDAO.selectAllOrdered(tableSchema);
timer.logTimeTaken("selectAllOrdered");

rf2FileWriter.exportFullAndSnapshot(fullResultSet, tableSchema,
build.getConfiguration().getEffectiveTime(), fullFileAsyncPipe.getOutputStream(),
snapshotAsyncPipe.getOutputStream());
LOGGER.debug("Completed processing full and snapshot files for {}, waiting for network.", tableSchema.getTableName());
fullFileAsyncPipe.waitForFinish();
snapshotAsyncPipe.waitForFinish();
break;
}
}
} catch (final Exception e) {
final String errorMsg = "Failed to generate subsequent full and snapshot release files due to: " + ExceptionUtils.getRootCauseMessage(e);
throw new ReleaseFileGenerationException(errorMsg, e);
} finally {
// Clean up time
if (rf2TableDAO != null) {
try {
rf2TableDAO.closeConnection();
} catch (final Exception e) {
LOGGER.error("Failure while trying to clean up after {}", tableSchema != null ? tableSchema.getTableName() : "No table yet.", e);
}
}
}
}


private InputStream getEquivalentAdditionalPackageInputStream(String additionalPreviousPublishedPackage, String currentAdditionalFileName) throws IOException {
return buildDao.getPublishedFileArchiveEntry(EXTERNAL_DEPENDENCY_PACKAGE_RELEASE_CENTER, getEquivalentAdditionalFile(additionalPreviousPublishedPackage, currentAdditionalFileName), additionalPreviousPublishedPackage);
}

private InputStream getEquivalentInternationalFull(ExtensionConfig extensionConfig, String transformedDeltaDataFile) throws IOException {
String equivalentFullFile = getEquivalentInternationalFile(extensionConfig, transformedDeltaDataFile).replace(DELTA, FULL);
LOGGER.info("Equivalent full file {}", equivalentFullFile);
Expand Down Expand Up @@ -287,6 +441,17 @@ private String getEquivalentInternationalFile(ExtensionConfig extensionConfig, S
return equivalentBuilder.toString();
}

private String getEquivalentAdditionalFile(String additionalRelease, String additionalFilename) {
additionalFilename = additionalFilename.replace(RF2Constants.TXT_FILE_EXTENSION, "");
if (configuration.isBetaRelease() && additionalFilename.startsWith(BETA_PREFIX)) {
additionalFilename = additionalFilename.substring(1);
}
String[] splits = additionalFilename.split(RF2Constants.FILE_NAME_SEPARATOR);
splits[splits.length - 1] = RF2BuildUtils.getReleaseDateFromReleasePackage(additionalRelease);
LOGGER.info("The equivalent file for {} in additional package is {}", additionalFilename, String.join(RF2Constants.FILE_NAME_SEPARATOR, splits) + RF2Constants.TXT_FILE_EXTENSION);
return String.join(RF2Constants.FILE_NAME_SEPARATOR, splits) + RF2Constants.TXT_FILE_EXTENSION;
}

private InputStream getPreviousFileStream(final String previousPublishedPackage, final String currentFileName) throws IOException {
final InputStream previousFileStream = buildDao.getPublishedFileArchiveEntry(build.getReleaseCenterKey(), currentFileName, previousPublishedPackage);
if (previousFileStream == null) {
Expand Down
Loading

0 comments on commit b2283b3

Please sign in to comment.