Merge branch '81-transform-gui-to-nightsky-api' into 'master'

Resolve "Transform GUI to Nightsky API" Closes #81, #123, #87, #84, #83, #60, #44, #42, #27, #16, #13, and #11 See merge request bright-giant/sirius/sirius-frontend!26
sirius-ms · Mar 7, 2024 · 0813649 · 0813649
2 parents a28e88c + bcc5d55
commit 0813649
Show file tree

Hide file tree

Showing 627 changed files with 62,638 additions and 23,265 deletions.
diff --git a/build.gradle b/build.gradle
@@ -29,10 +29,14 @@ wrapper {
 
 //Global Properties
 ext {
+    junit_version = '5.10.0'
+    jackson_version = "2.16.1"
+    spring_boot_version = "3.2.3"
+    springdoc_version = "2.3.0"
     //version for external libraries
     slf4j_version = '1.7.36'
-    jjobs_version = '0.9.35'
-
+    jjobs_version = '0.9.36-SNAPSHOT'
+    jaxb_version = "2.3.9"
     cdk_version = "2.8"
 
     picocli_version = '4.6.3'//'3.9.5'
@@ -145,9 +149,19 @@ allprojects {
 
         //NOTE: cannot be provided as transitive dependency
         implementation "jakarta.persistence:jakarta.persistence-api:$jakarta_version"
+        implementation "org.glassfish.jaxb:jaxb-runtime:$jaxb_version"
+
+        // Aggregator dependency that also brings JUnit 5 parameterized tests etc.
+        testImplementation("org.junit.jupiter:junit-jupiter:$junit_version")
+        testImplementation("org.junit.jupiter:junit-jupiter-params:$junit_version")
+        // The Vintage engine is needed to be able to run JUnit 4 tests
+        testImplementation("org.junit.vintage:junit-vintage-engine:$junit_version")
+        testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junit_version"
+    }
 
-
-        testImplementation group: 'junit', name: 'junit', version: '4.12'
+    test {
+        useJUnitPlatform()
+        maxHeapSize = "16G"
     }
 
     //there are some outdated incompatible trove dependencies in some libs.

diff --git a/settings.gradle b/settings.gradle
@@ -5,6 +5,7 @@ include ':sirius_dist'
     include ':sirius_dist:sirius_gui_dist'
 include ':sirius_gui'
 include ':sirius_rest_service'
-//include ':sirius_utils'
-include 'sirius_doc'
+include ':sirius_nightsky_sdk'
+include ':sirius_nightsky_sdk:sirius_nightsky_sdk.openapi'
+include 'sirius_nightsky_sdk:sirius_nightsky_sdk.jjobs'
 
diff --git a/sirius_cli/build.gradle b/sirius_cli/build.gradle
@@ -1,5 +1,7 @@
 apply plugin: 'java-library'
 dependencies {
+    compileOnly 'org.projectlombok:lombok:1.18.26'
+    annotationProcessor 'org.projectlombok:lombok:1.18.26'
     // wrong place START (should be in dist) but IDEA does not recognize dependency from no code modules //todo find alternative
     implementation group: 'org.slf4j', name: 'slf4j-jdk14', version: "2.0.7"
     implementation("de.unijena.bioinf.cbc-java:cbc-java-jni:$cbc_version:${siriusTargetPlatform.jenaClassifier().orElseThrow()}")

diff --git a/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/BackgroundRuns.java b/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/BackgroundRuns.java
diff --git a/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/CLIRootOptions.java b/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/CLIRootOptions.java
@@ -20,12 +20,12 @@
 package de.unijena.bioinf.ms.frontend.subtools;
 
 import de.unijena.bioinf.ChemistryBase.jobs.SiriusJobs;
-import de.unijena.bioinf.rest.NetUtils;
 import de.unijena.bioinf.jjobs.JJob;
 import de.unijena.bioinf.ms.frontend.core.ApplicationCore;
 import de.unijena.bioinf.ms.frontend.subtools.config.DefaultParameterConfigLoader;
 import de.unijena.bioinf.ms.properties.PropertyManager;
 import de.unijena.bioinf.projectspace.*;
+import de.unijena.bioinf.rest.NetUtils;
 import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -56,7 +56,7 @@ public class CLIRootOptions<I extends Instance, M extends ProjectSpaceManager<I>
     public static final Logger LOG = LoggerFactory.getLogger(CLIRootOptions.class);
 
     protected final DefaultParameterConfigLoader defaultConfigOptions;
-    protected final ProjectSpaceManagerFactory<I, M> spaceManagerFactory;
+    protected ProjectSpaceManagerFactory<I, M> spaceManagerFactory;
 
     public CLIRootOptions(@NotNull DefaultParameterConfigLoader defaultConfigOptions, @NotNull ProjectSpaceManagerFactory<I, M> spaceManagerFactory) {
         this.defaultConfigOptions = defaultConfigOptions;
@@ -156,6 +156,10 @@ public ProjectSpaceManagerFactory<I, M> getSpaceManagerFactory() {
         return spaceManagerFactory;
     }
 
+    public void setSpaceManagerFactory(ProjectSpaceManagerFactory<I, M> spaceManagerFactory) {
+        this.spaceManagerFactory = spaceManagerFactory;
+    }
+
     @Override
     public M getProjectSpace() {
         if (projectSpaceToWriteOn == null)

diff --git a/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/ComputeRootOption.java b/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/ComputeRootOption.java
@@ -46,7 +46,7 @@ public ComputeRootOption(@NotNull P projectSpace) {
     }
 
     public ComputeRootOption(@NotNull P projectSpace, @NotNull InputFilesOptions inputFiles) {
-        this(projectSpace, null, inputFiles);
+        this(projectSpace, (Iterable<I>) null, inputFiles);
     }
 
     public ComputeRootOption(@NotNull P projectSpace, @Nullable Iterable<I> instances) {
@@ -60,6 +60,10 @@ public ComputeRootOption(@NotNull P projectSpace, @Nullable Iterable<I> instance
     }
 
     public ComputeRootOption(@NotNull P projectSpace, @Nullable List<CompoundContainerId> containerIds) {
+        this(projectSpace, containerIds, null);
+
+    }
+    public ComputeRootOption(@NotNull P projectSpace, @Nullable List<CompoundContainerId> containerIds, @Nullable InputFilesOptions inputFiles) {
         this.projectSpace = projectSpace;
 
         instances = containerIds != null ? new IterableWithSize<>() {
@@ -74,7 +78,7 @@ public Iterator<I> iterator() {
                 return makeInstanceIterator(containerIds.iterator());
             }
         } : null;
-        this.inputFiles = null;
+        this.inputFiles = inputFiles;
     }
 
     /**

diff --git a/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/DataSetJob.java b/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/DataSetJob.java
@@ -23,6 +23,7 @@
 import de.unijena.bioinf.jjobs.JJob;
 import de.unijena.bioinf.jjobs.JobSubmitter;
 import de.unijena.bioinf.projectspace.Instance;
+import de.unijena.bioinf.projectspace.ProjectSpaceManager;
 import org.jetbrains.annotations.NotNull;
 
 import java.util.ArrayList;
@@ -110,6 +111,11 @@ protected void checkInputs() {
             );
     }
 
+    @Override
+    public String getProjectName() {
+        return (inputInstances != null ? inputInstances.stream().map(Instance::getProjectSpaceManager).map(ProjectSpaceManager::getName).findAny().orElse("N/A") : "<Awaiting Instance>");
+    }
+
     @Override
     protected void cleanup() {
         super.cleanup();

diff --git a/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/InstanceJob.java b/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/InstanceJob.java
@@ -103,6 +103,11 @@ public String identifier() {
         return super.identifier() + " | " + (input != null ? input.toString() : "<Awaiting Instance>");
     }
 
+    @Override
+    public String getProjectName() {
+        return (input != null ? input.getProjectSpaceManager().getName() : "<Awaiting Instance>");
+    }
+
     /**
      * Check if the input is valid for computation. May be overwritten by implementations for additional checks.
      * @return false if input data is fine and true if data should be skipped gently. IllegalArgumentException is thrown

diff --git a/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/ToolChainJob.java b/sirius_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/ToolChainJob.java
@@ -52,6 +52,7 @@ default boolean setRecompute(final @NotNull Instance inst, boolean recompute) {
     default String getToolName() {
         return getClass().getSimpleName();
     }
+    String getProjectName();
 
     void setInvalidator(Consumer<Instance> invalidator);
 

diff --git a/...s_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/custom_db/CustomDBOptions.java b/...s_cli/src/main/java/de/unijena/bioinf/ms/frontend/subtools/custom_db/CustomDBOptions.java
@@ -19,14 +19,12 @@
 
 package de.unijena.bioinf.ms.frontend.subtools.custom_db;
 
+import de.unijena.bioinf.ChemistryBase.fp.CdkFingerprintVersion;
 import de.unijena.bioinf.ChemistryBase.jobs.SiriusJobs;
 import de.unijena.bioinf.ChemistryBase.utils.FileUtils;
 import de.unijena.bioinf.babelms.MsExperimentParser;
-import de.unijena.bioinf.chemdb.SearchableDatabases;
-import de.unijena.bioinf.chemdb.custom.CustomDatabase;
-import de.unijena.bioinf.chemdb.custom.CustomDatabaseFactory;
-import de.unijena.bioinf.chemdb.custom.CustomDatabaseImporter;
-import de.unijena.bioinf.chemdb.custom.CustomDatabaseSettings;
+import de.unijena.bioinf.babelms.inputresource.PathInputResource;
+import de.unijena.bioinf.chemdb.custom.*;
 import de.unijena.bioinf.jjobs.BasicMasterJJob;
 import de.unijena.bioinf.jjobs.JJob;
 import de.unijena.bioinf.ms.frontend.core.ApplicationCore;
@@ -38,16 +36,17 @@
 import de.unijena.bioinf.ms.frontend.workflow.Workflow;
 import de.unijena.bioinf.ms.properties.ParameterConfig;
 import de.unijena.bioinf.ms.rest.model.info.VersionsInfo;
-import de.unijena.bioinf.storage.blob.Compressible;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 import org.slf4j.LoggerFactory;
 import picocli.CommandLine;
 import picocli.CommandLine.Option;
 
-import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.*;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
@@ -80,10 +79,26 @@ public static class Import {
                 description = {"Location of the custom database to import into.",
                         "An absolute local path to a new database file file to be created (file name must end with .db)",
                         "If no input data is given (--input), the database will just be added to SIRIUS",
-                        "The added db will also be available in the GUI."
-                }, order = 201)
+                        "The added db will also be available in the GUI."}, order = 201)
         String location = null;
 
+        @Option(names = "--name", order = 202,
+                description = {"Name/Identifier of the custom database.",
+                        "If not given filename from location will be used."})
+        String name = null;
+
+        @Option(names = "--displayName", order = 203,
+                description = {"Displayable name of the custom database.",
+                        "This is the preferred name to be shown in the GUI. Maximum Length: 15 characters.",
+                        "If not given name will be used."})
+        public void setDisplayName(String displayName) {
+            if (displayName.length() > 15)
+                throw new CommandLine.PicocliException("Maximum allowed length for display names is 15 characters.");
+            this.displayName = displayName;
+        }
+
+        String displayName = null;
+
         @Option(names = {"--buffer-size", "--buffer"}, defaultValue = "1000",
                 description = {"Maximum number of downloaded/computed compounds to keep in memory before writing them to disk (into the db directory). Can be set higher when importing large files on a fast computer."},
                 order = 210)
@@ -148,6 +163,9 @@ public void run() {
 
         @Override
         protected Boolean compute() throws Exception {
+            final CdkFingerprintVersion version = ApplicationCore.WEB_API.getCDKChemDBFingerprintVersion();
+            //loads all current available dbs
+            final @NotNull List<CustomDatabase> dbs = CustomDatabases.getCustomDatabases(version);
 
             if (mode.importParas != null) {
                 if (mode.importParas.location == null || mode.importParas.location.isBlank()) {
@@ -157,13 +175,20 @@ protected Boolean compute() throws Exception {
 
                 checkForInterruption();
 
-                checkConflictingName(mode.importParas.location);
+                checkConflictingName(mode.importParas.location, mode.importParas.name);
 
-                CustomDatabaseSettings settings = new CustomDatabaseSettings(false, 0,
-                        List.of(ApplicationCore.WEB_API.getCDKChemDBFingerprintVersion().getUsedFingerprints()), VersionsInfo.CUSTOM_DATABASE_SCHEMA, null);
+                CustomDatabaseSettings settings = CustomDatabaseSettings.builder()
+                        .usedFingerprints(List.of(version.getUsedFingerprints()))
+                        .schemaVersion(VersionsInfo.CUSTOM_DATABASE_SCHEMA)
+                        .name(mode.importParas.name)
+                        .displayName(mode.importParas.displayName)
+                        .matchRtOfReferenceSpectra(false)
+                        .statistics(new CustomDatabaseSettings.Statistics())
+                        .build();
+
+                final CustomDatabase db = CustomDatabases.createOrOpen(mode.importParas.location, settings, version);
+                writeDBProperties();
 
-                final CustomDatabase db = CustomDatabaseFactory.createOrOpen(mode.importParas.location, Compressible.Compression.NONE, settings);
-                addDBToPropertiesIfNotExist(db);
                 logInfo("Database added to SIRIUS. Use 'structure --db=\"" + db.storageLocation() + "\"' to search in this database.");
 
                 if (mode.importParas.input == null || mode.importParas.input.isEmpty())
@@ -189,8 +214,8 @@ protected Boolean compute() throws Exception {
                 checkForInterruption();
 
                 dbjob = db.importToDatabaseJob(
-                        spectrumFiles.stream().map(Path::toFile).toList(),
-                        structureFiles.stream().map(Path::toFile).toList(),
+                        spectrumFiles.stream().map(PathInputResource::new).collect(Collectors.toList()),
+                        structureFiles.stream().map(PathInputResource::new).collect(Collectors.toList()),
                         inChI -> updateProgress(0, Math.max(lines.intValue(), count.incrementAndGet() + 1), count.get(), "Importing '" + inChI.key2D() + "'"),
                         ApplicationCore.WEB_API, mode.importParas.writeBuffer
                 );
@@ -200,25 +225,19 @@ protected Boolean compute() throws Exception {
 
                 return true;
             } else if (mode.removeParas != null) {
-                if (mode.removeParas.location == null ||mode.removeParas.location.isBlank())
+                if (mode.removeParas.location == null || mode.removeParas.location.isBlank())
                     throw new IllegalArgumentException("Database location to remove not specified!");
 
-                SearchableDatabases.getCustomDatabase(mode.removeParas.location).ifPresentOrElse(db -> {
-                            removeDBFromProperties(db);
-                            if (mode.removeParas.delete) {
-                                try {
-                                    CustomDatabaseFactory.delete(db);
-                                } catch (IOException e) {
-                                    logError("Error deleting database.", e);
-                                }
-                            }
-                        }, () -> logWarn("\n==> No custom database with location '" + mode.removeParas.location + "' found.\n")
-                );
+                CustomDatabases.getCustomDatabase(mode.removeParas.location, version)
+                        .ifPresentOrElse(db -> {
+                                    CustomDatabases.remove(db, mode.removeParas.delete);
+                                    writeDBProperties();
+                                }, () -> logWarn("\n==> No custom database with location '" + mode.removeParas.location + "' found.\n")
+                        );
                 return true;
             } else if (mode.showParas != null) {
                 if (mode.showParas.db == null) {
-                    @NotNull List<CustomDatabase> dbs = SearchableDatabases.getCustomDatabases();
-                    if (dbs.isEmpty()){
+                    if (dbs.isEmpty()) {
                         logWarn("\n==> No Custom database found!\n");
                         return false;
                     }
@@ -230,7 +249,7 @@ protected Boolean compute() throws Exception {
                     });
                     return true;
                 } else {
-                    SearchableDatabases.getCustomDatabase(mode.showParas.db)
+                    CustomDatabases.getCustomDatabase(mode.showParas.db, version)
                             .ifPresentOrElse(
                                     CustomDBOptions.this::printDBInfo,
                                     () -> logWarn("\n==> No custom database with location '" + mode.showParas.db + "' found.\n"));
@@ -254,6 +273,7 @@ private void printDBInfo(CustomDatabase db) {
         CustomDatabaseSettings s = db.getSettings();
         System.out.println("##########  BEGIN DB INFO  ##########");
         System.out.println("Name: " + db.name());
+        System.out.println("Display Name: " + db.displayName());
         System.out.println("Location: " + db.storageLocation());
         System.out.println("Number of Formulas: " + s.getStatistics().getFormulas());
         System.out.println("Number of Structures: " + s.getStatistics().getCompounds());
@@ -262,40 +282,28 @@ private void printDBInfo(CustomDatabase db) {
             System.out.println("Version: " + db.getDatabaseVersion());
             System.out.println("Schema Version: " + s.getSchemaVersion());
             System.out.println("FilterFlag: " + db.getFilterFlag());
-            System.out.println("Used Fingerprints: [ '" + s.getFingerprintVersion().stream().map(Enum::name).collect(Collectors.joining("','")) + "' ]");
+            System.out.println("Used Fingerprints: [ '" + s.getUsedFingerprints().stream().map(Enum::name).collect(Collectors.joining("','")) + "' ]");
         }
         System.out.println("###############  END  ###############");
     }
 
-    public static void addDBToPropertiesIfNotExist(@NotNull CustomDatabase db) {
-        Set<CustomDatabase> customs = new HashSet<>(SearchableDatabases.getCustomDatabases());
-        if (!customs.contains(db)) {
-            customs.add(db);
-            writeDBProperties(customs);
-        }
-    }
-
-    public static void removeDBFromProperties(@NotNull CustomDatabase db) {
-        Set<CustomDatabase> customs = new HashSet<>(SearchableDatabases.getCustomDatabases());
-        customs.remove(db);
-        writeDBProperties(customs);
-    }
-
-    private static void writeDBProperties(Collection<CustomDatabase> dbs) {
-        SiriusProperties.SIRIUS_PROPERTIES_FILE().setAndStoreProperty(SearchableDatabases.PROP_KEY, dbs.stream()
-                .sorted(Comparator.comparing(CustomDatabase::name))
-                .map(CustomDatabase::storageLocation)
+    public static void writeDBProperties() {
+        SiriusProperties.SIRIUS_PROPERTIES_FILE().setAndStoreProperty(CustomDataSources.PROP_KEY, CustomDataSources.sourcesStream()
+                .filter(CustomDataSources.Source::isCustomSource)
+                .map(c -> (CustomDataSources.CustomSource) c)
+                .sorted(Comparator.comparing(CustomDataSources.Source::name))
+                .map(CustomDataSources.CustomSource::location)
                 .collect(Collectors.joining(",")));
     }
 
-    private static void checkConflictingName(String location) {
-        String dbName = Path.of(location).getFileName().toString();
-        SearchableDatabases.getCustomDatabases().stream()
-                .filter(db -> db.name().equals(dbName) && !db.storageLocation().equals(location))
+    private static void checkConflictingName(@NotNull String location, @Nullable String name) {
+        final String dbName = name != null ? name : Path.of(location).getFileName().toString();
+        CustomDataSources.sourcesStream().filter(CustomDataSources.Source::isCustomSource)
+                .filter(db -> db.name().equals(dbName) && !location.equals(db.isCustomSource() ? ((CustomDataSources.CustomSource) db).location() : null))
                 .findAny()
                 .ifPresent(db -> {
-                    throw new RuntimeException("Database with name " + dbName + " already exists in " + db.storageLocation());
-                } );
+                    throw new RuntimeException("Database with name " + dbName + " already exists in " /*+ db.id()*/);
+                });
     }
 
 }
diff --git a/...i/src/main/java/de/unijena/bioinf/ms/frontend/subtools/export/mgf/MgfExporterOptions.java b/...i/src/main/java/de/unijena/bioinf/ms/frontend/subtools/export/mgf/MgfExporterOptions.java
@@ -60,13 +60,6 @@ public void setMergePpm(DefaultParameter value) throws Exception {
         this.ppmDev = value.asDouble();
     }
 
-    //fewer parameter is probably better // todo hidden then?
-//    @CommandLine.Option(names = "--merge-abs", description = "Maximum allowed absolute difference for peaks of MS2 spectra to be merged.", defaultValue = "0.005")
-//    public void setMergeAbs(DefaultParameter value) throws Exception {
-//        this.absDev = value.asDouble();
-//    }
-
-
     @CommandLine.Option(names = {"--output", "-o"}, description = "Specify the mgf file destination.")
     public void setOutput(String outputPath) {
         output = Paths.get(outputPath);