Added get-import-status

DANS-KNAW · Nov 10, 2024 · e525c33 · e525c33
1 parent 68f3363
commit e525c33
Show file tree

Hide file tree

Showing 4 changed files with 41 additions and 14 deletions.
diff --git a/src/main/java/nl/knaw/dans/dvingest/core/Deposit.java b/src/main/java/nl/knaw/dans/dvingest/core/Deposit.java
@@ -31,6 +31,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collections;
+import java.util.List;
 import java.util.UUID;
 
 @Getter
@@ -51,13 +52,28 @@ public Dataset getDatasetMetadata() throws IOException {
         SimpleModule module = new SimpleModule();
         module.addDeserializer(MetadataField.class, new MetadataFieldDeserializer());
         mapper.registerModule(module);
-        var dataset = mapper.readValue(FileUtils.readFileToString(location.resolve("dataset.yml").toFile(), "UTF-8"), Dataset.class);
+        var dataset = mapper.readValue(FileUtils.readFileToString(getBagDir().resolve("dataset.yml").toFile(), "UTF-8"), Dataset.class);
         dataset.getDatasetVersion().setFiles(Collections.emptyList()); // files = null or a list of files is not allowed
         return dataset;
     }
 
+    public Path getBagDir() {
+        try (var files = Files.list(location).filter(Files::isDirectory)) {
+            List<Path> filesList = files.toList();
+            if (filesList.size() == 1) {
+                return filesList.get(0);
+            }
+            else {
+                throw new IllegalStateException("Deposit " + location + " should contain exactly one directory");
+            }
+        }
+        catch (IOException e) {
+            throw new IllegalStateException("Error listing files in deposit " + location, e);
+        }
+    }
+
     public Path getFilesDir() {
-        return location.resolve("files");
+        return getBagDir().resolve("data");
     }
 
     public void moveTo(Path targetDir) throws IOException {

diff --git a/src/main/java/nl/knaw/dans/dvingest/core/ImportJob.java b/src/main/java/nl/knaw/dans/dvingest/core/ImportJob.java
@@ -62,21 +62,22 @@ public void run() {
                 deposits.add(new Deposit(Path.of(importCommand.getPath())));
             }
             else {
-                // Multiple objects
-                // Create deposit for each object
-                // Add to list
+                try (var depositPaths = Files.list(Path.of(importCommand.getPath()))) {
+                    depositPaths.forEach(p -> deposits.add(new Deposit(p)));
+                }
             }
 
             initOutputDir();
 
             // Process deposits
             for (Deposit deposit : deposits) {
+                log.info("START Processing deposit: {}", deposit.getId());
                 new IngestTask(deposit, dataverseClient, outputDir).run();
+                log.info("END Processing deposit: {}", deposit.getId());
                 // TODO: record number of processed/rejected/failed deposits in ImportJob status
             }
 
-            // Job completed, some deposits may still have failed, TODO: change to DONE
-            status.setStatus(StatusEnum.SUCCESS);
+            status.setStatus(StatusEnum.DONE);
         }
         catch (Exception e) {
             log.error("Failed to process import job", e);

diff --git a/src/main/java/nl/knaw/dans/dvingest/core/IngestArea.java b/src/main/java/nl/knaw/dans/dvingest/core/IngestArea.java
@@ -25,7 +25,9 @@
 
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 
 @Slf4j
@@ -40,7 +42,7 @@ public class IngestArea {
     @NonNull
     private final Path outbox;
 
-    private final Map<String, ImportJob> importJobs = new java.util.concurrent.ConcurrentHashMap<>();
+    private final Map<String, ImportJob> importJobs = new ConcurrentHashMap<>();
 
     private IngestArea(ExecutorService executorService, DataverseClient dataverseClient, Path inbox, Path outbox) {
         try {
@@ -68,12 +70,15 @@ public void submit(ImportCommandDto importCommand) {
         executorService.submit(importJob);
     }
 
-    public ImportJobStatusDto getStatus(String path) {
-        var importJob = importJobs.get(path);
-        if (importJob == null) {
-            throw new IllegalArgumentException("No job for " + path);
+    public List<ImportJobStatusDto> getStatus(String path) {
+        if (path == null) {
+            return importJobs.values().stream().map(ImportJob::getStatus).toList();
+        } else {
+            if (importJobs.get(path) == null) {
+                throw new IllegalArgumentException("No job found for path: " + path);
+            }
+            return List.of(importJobs.get(path).getStatus());
         }
-        return importJob.getStatus();
     }
 
     private ImportJob createImportJob(ImportCommandDto importCommand) {

diff --git a/src/main/java/nl/knaw/dans/dvingest/resources/IngestApiResource.java b/src/main/java/nl/knaw/dans/dvingest/resources/IngestApiResource.java
@@ -25,6 +25,11 @@
 public class IngestApiResource implements IngestApi {
     private final IngestArea ingestArea;
 
+    @Override
+    public Response ingestGet(String path) {
+        return Response.ok(ingestArea.getStatus(path)).build();
+    }
+
     @Override
     public Response ingestPost(ImportCommandDto importCommandDto) {
         try {
@@ -33,6 +38,6 @@ public Response ingestPost(ImportCommandDto importCommandDto) {
         catch (IllegalArgumentException e) {
             return Response.status(Response.Status.BAD_REQUEST).entity(e.getMessage()).build();
         }
-        return Response.ok(ingestArea.getStatus(importCommandDto.getPath())).build();
+        return Response.ok(ingestArea.getStatus(importCommandDto.getPath()).get(0)).build();
     }
 }