diff --git a/src/main/java/nl/knaw/dans/dvingest/core/Deposit.java b/src/main/java/nl/knaw/dans/dvingest/core/Deposit.java index b22d618..0286d25 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/Deposit.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/Deposit.java @@ -31,6 +31,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Collections; +import java.util.List; import java.util.UUID; @Getter @@ -51,13 +52,28 @@ public Dataset getDatasetMetadata() throws IOException { SimpleModule module = new SimpleModule(); module.addDeserializer(MetadataField.class, new MetadataFieldDeserializer()); mapper.registerModule(module); - var dataset = mapper.readValue(FileUtils.readFileToString(location.resolve("dataset.yml").toFile(), "UTF-8"), Dataset.class); + var dataset = mapper.readValue(FileUtils.readFileToString(getBagDir().resolve("dataset.yml").toFile(), "UTF-8"), Dataset.class); dataset.getDatasetVersion().setFiles(Collections.emptyList()); // files = null or a list of files is not allowed return dataset; } + public Path getBagDir() { + try (var files = Files.list(location).filter(Files::isDirectory)) { + List filesList = files.toList(); + if (filesList.size() == 1) { + return filesList.get(0); + } + else { + throw new IllegalStateException("Deposit " + location + " should contain exactly one directory"); + } + } + catch (IOException e) { + throw new IllegalStateException("Error listing files in deposit " + location, e); + } + } + public Path getFilesDir() { - return location.resolve("files"); + return getBagDir().resolve("data"); } public void moveTo(Path targetDir) throws IOException { diff --git a/src/main/java/nl/knaw/dans/dvingest/core/ImportJob.java b/src/main/java/nl/knaw/dans/dvingest/core/ImportJob.java index 4e25a0d..f5e4d02 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/ImportJob.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/ImportJob.java @@ -62,21 +62,22 @@ public void run() { deposits.add(new Deposit(Path.of(importCommand.getPath()))); } else { - // Multiple objects - // Create deposit for each object - // Add to list + try (var depositPaths = Files.list(Path.of(importCommand.getPath()))) { + depositPaths.forEach(p -> deposits.add(new Deposit(p))); + } } initOutputDir(); // Process deposits for (Deposit deposit : deposits) { + log.info("START Processing deposit: {}", deposit.getId()); new IngestTask(deposit, dataverseClient, outputDir).run(); + log.info("END Processing deposit: {}", deposit.getId()); // TODO: record number of processed/rejected/failed deposits in ImportJob status } - // Job completed, some deposits may still have failed, TODO: change to DONE - status.setStatus(StatusEnum.SUCCESS); + status.setStatus(StatusEnum.DONE); } catch (Exception e) { log.error("Failed to process import job", e); diff --git a/src/main/java/nl/knaw/dans/dvingest/core/IngestArea.java b/src/main/java/nl/knaw/dans/dvingest/core/IngestArea.java index d003b42..e36723e 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/IngestArea.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/IngestArea.java @@ -25,7 +25,9 @@ import java.io.IOException; import java.nio.file.Path; +import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @Slf4j @@ -40,7 +42,7 @@ public class IngestArea { @NonNull private final Path outbox; - private final Map importJobs = new java.util.concurrent.ConcurrentHashMap<>(); + private final Map importJobs = new ConcurrentHashMap<>(); private IngestArea(ExecutorService executorService, DataverseClient dataverseClient, Path inbox, Path outbox) { try { @@ -68,12 +70,15 @@ public void submit(ImportCommandDto importCommand) { executorService.submit(importJob); } - public ImportJobStatusDto getStatus(String path) { - var importJob = importJobs.get(path); - if (importJob == null) { - throw new IllegalArgumentException("No job for " + path); + public List getStatus(String path) { + if (path == null) { + return importJobs.values().stream().map(ImportJob::getStatus).toList(); + } else { + if (importJobs.get(path) == null) { + throw new IllegalArgumentException("No job found for path: " + path); + } + return List.of(importJobs.get(path).getStatus()); } - return importJob.getStatus(); } private ImportJob createImportJob(ImportCommandDto importCommand) { diff --git a/src/main/java/nl/knaw/dans/dvingest/resources/IngestApiResource.java b/src/main/java/nl/knaw/dans/dvingest/resources/IngestApiResource.java index e77bf96..5611660 100644 --- a/src/main/java/nl/knaw/dans/dvingest/resources/IngestApiResource.java +++ b/src/main/java/nl/knaw/dans/dvingest/resources/IngestApiResource.java @@ -25,6 +25,11 @@ public class IngestApiResource implements IngestApi { private final IngestArea ingestArea; + @Override + public Response ingestGet(String path) { + return Response.ok(ingestArea.getStatus(path)).build(); + } + @Override public Response ingestPost(ImportCommandDto importCommandDto) { try { @@ -33,6 +38,6 @@ public Response ingestPost(ImportCommandDto importCommandDto) { catch (IllegalArgumentException e) { return Response.status(Response.Status.BAD_REQUEST).entity(e.getMessage()).build(); } - return Response.ok(ingestArea.getStatus(importCommandDto.getPath())).build(); + return Response.ok(ingestArea.getStatus(importCommandDto.getPath()).get(0)).build(); } }