Skip to content

Commit

Permalink
Merge branch '43-implement-reporting-of-affected-compounds-and-featur…
Browse files Browse the repository at this point in the history
…es-in-jobs' into 'master'

Resolve "implement reporting of affected compounds and features in jobs and compute flag handling"

Closes #37 and #43

See merge request bright-giant/sirius/sirius-frontend!15
  • Loading branch information
Markus Fleischauer committed Oct 29, 2023
2 parents c9b8e88 + c923b21 commit 25dd696
Show file tree
Hide file tree
Showing 12 changed files with 244 additions and 114 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -25,24 +25,25 @@
import de.unijena.bioinf.ms.frontend.subtools.ComputeRootOption;
import de.unijena.bioinf.ms.frontend.subtools.InputFilesOptions;
import de.unijena.bioinf.ms.frontend.subtools.config.DefaultParameterConfigLoader;
import de.unijena.bioinf.ms.frontend.subtools.projectspace.ImportFromMemoryWorkflow;
import de.unijena.bioinf.ms.frontend.subtools.projectspace.ProjectSpaceWorkflow;
import de.unijena.bioinf.ms.frontend.workflow.*;
import de.unijena.bioinf.ms.properties.PropertyManager;
import de.unijena.bioinf.projectspace.CompoundContainerId;
import de.unijena.bioinf.projectspace.Instance;
import de.unijena.bioinf.projectspace.ProjectSpaceManager;
import de.unijena.bioinf.projectspace.*;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.beans.PropertyChangeSupport;
import java.io.BufferedReader;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiConsumer;
import java.util.function.Supplier;

/**
* Manage and execute command line (toolchain) runs in the background as if you would have started it via the CLI.
Expand Down Expand Up @@ -164,6 +165,14 @@ public static <P extends ProjectSpaceManager<I>, I extends Instance> BackgroundR
);
}

public static <P extends ProjectSpaceManager<I>, I extends Instance> BackgroundRunJob<P, I> runImport(
@NotNull P project, Supplier<BufferedReader> data, @Nullable String source, @NotNull String ext
) {
Workflow computation = new ImportFromMemoryWorkflow(project, data, source, ext);
return SiriusJobs.getGlobalJobManager().submitJob(
new BackgroundRunJob<>(computation, project, (Iterable<I>) null, RUN_COUNTER.incrementAndGet(), null));
}

private static <P extends ProjectSpaceManager<I>, I extends Instance> Workflow makeWorkflow(
List<String> command, ComputeRootOption<P, I> rootOptions) throws IOException {
final DefaultParameterConfigLoader configOptionLoader = new DefaultParameterConfigLoader(PropertyManager.DEFAULTS.newIndependentInstance("BATCH_COMPUTE"));
Expand Down Expand Up @@ -279,6 +288,10 @@ protected void cleanup() {
logInfo("Collecting imported compounds...");
instanceIds = ((ProjectSpaceWorkflow) computation).getImportedCompounds();
logInfo("Imported compounds collected...");
} else if (computation instanceof ImportFromMemoryWorkflow) {
logInfo("Collecting imported compounds...");
instanceIds = ((ImportFromMemoryWorkflow) computation).getImportedCompounds();
logInfo("Imported compounds collected...");
}
logInfo("Freeing up memory...");
computation = null;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
/*
*
* This file is part of the SIRIUS library for analyzing MS and MS/MS data
*
* Copyright (C) 2013-2020 Kai Dührkop, Markus Fleischauer, Marcus Ludwig, Martin A. Hoffman, Fleming Kretschmer and Sebastian Böcker,
* Chair of Bioinformatics, Friedrich-Schiller University.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License along with SIRIUS. If not, see <https://www.gnu.org/licenses/lgpl-3.0.txt>
*/

package de.unijena.bioinf.ms.frontend.subtools.projectspace;

import de.unijena.bioinf.ChemistryBase.ms.Ms2Experiment;
import de.unijena.bioinf.ChemistryBase.ms.SpectrumFileSource;
import de.unijena.bioinf.babelms.CloseableIterator;
import de.unijena.bioinf.babelms.GenericParser;
import de.unijena.bioinf.babelms.MsExperimentParser;
import de.unijena.bioinf.jjobs.JobProgressEvent;
import de.unijena.bioinf.jjobs.JobProgressEventListener;
import de.unijena.bioinf.jjobs.JobProgressMerger;
import de.unijena.bioinf.jjobs.ProgressSupport;
import de.unijena.bioinf.ms.frontend.workflow.Workflow;
import de.unijena.bioinf.projectspace.CompoundContainerId;
import de.unijena.bioinf.projectspace.Instance;
import de.unijena.bioinf.projectspace.ProjectSpaceManager;
import org.jetbrains.annotations.NotNull;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Supplier;

public class ImportFromMemoryWorkflow implements Workflow, ProgressSupport {
protected final JobProgressMerger progressSupport = new JobProgressMerger(this);
private List<CompoundContainerId> importedCompounds = null;

public List<CompoundContainerId> getImportedCompounds() {
return importedCompounds;
}

private ProjectSpaceManager<?> psm;

private Supplier<BufferedReader> dataReaderProvide;

private String sourceName;

private String ext;

public ImportFromMemoryWorkflow(ProjectSpaceManager<?> psm, Supplier<BufferedReader> dataReaderProvide, String sourceName, String ext) {
this.psm = psm;
this.dataReaderProvide = dataReaderProvide;
this.sourceName = sourceName;
this.ext = ext;
}

@Override
public void updateProgress(long min, long max, long progress, String shortInfo) {
progressSupport.updateConnectedProgress(min, max, progress, shortInfo);
}

@Override
public void addJobProgressListener(JobProgressEventListener listener) {
progressSupport.addPropertyChangeListener(listener);
}

@Override
public void removeJobProgressListener(JobProgressEventListener listener) {
progressSupport.removeProgress(listener);
}

@Override
public JobProgressEvent currentProgress() {
return progressSupport.currentConnectedProgress();
}

@Override
public JobProgressEvent currentCombinedProgress() {
return progressSupport.currentCombinedProgress();
}

@Override
public void run() {
importedCompounds = new ArrayList<>();
GenericParser<Ms2Experiment> parser = new MsExperimentParser()
.getParserByExt(ext);

int progress = 0;
try (BufferedReader bodyStream = dataReaderProvide.get()) {
updateProgress(0, -1, progress, "Data reader opened");
try (CloseableIterator<Ms2Experiment> it = parser.parseIterator(bodyStream, null)) {
while (it.hasNext()) {
Ms2Experiment next = it.next();
if (sourceName != null) // workaround to fake import file
next.setAnnotation(SpectrumFileSource.class,
new SpectrumFileSource(
new File("./" + (sourceName.endsWith(ext) ? sourceName : sourceName + "." + ext.toLowerCase())).toURI()));

@NotNull Instance inst = psm.newCompoundWithUniqueId(next);
importedCompounds.add(inst.getID());
updateProgress(0, -1, ++progress, "Imported: " + inst.getID().toString());
}
}
} catch (IOException e) {
throw new RuntimeException(e);
}
updateProgress(0, progress, progress, null);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@ public ProjectsProvider<?> projectsProvider(CLIRootOptions<?,?> cliRootOptions)
SiriusProjectSpaceProviderImpl projectsProvider = new SiriusProjectSpaceProviderImpl();
final SiriusProjectSpace ps = cliRootOptions.getProjectSpace().projectSpace();
projectsProvider.addProjectSpace(ps.getLocation().getFileName().toString(), ps);
System.out.println("added project to projectprovider");
return projectsProvider;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,6 @@ public static void main(String[] args) {

@Override
public void run(String... args) {
System.out.println("Start Middleware workflow with preprocessing");
RUN.compute();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,9 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import de.unijena.bioinf.ChemistryBase.utils.FileUtils;
import de.unijena.bioinf.ms.frontend.core.Workspace;
import de.unijena.bioinf.ms.middleware.SiriusContext;
import de.unijena.bioinf.ms.middleware.model.compute.ImportLocalFilesSubmission;
import de.unijena.bioinf.ms.middleware.model.compute.ImportStringSubmission;
import de.unijena.bioinf.ms.middleware.model.compute.JobId;
import de.unijena.bioinf.ms.middleware.model.compute.Job;
import de.unijena.bioinf.ms.middleware.model.compute.JobSubmission;
import de.unijena.bioinf.ms.middleware.service.compute.ComputeService;
import de.unijena.bioinf.ms.middleware.service.projects.Project;
Expand Down Expand Up @@ -74,9 +73,9 @@ public ComputeController(ComputeService<?> computeService, ProjectsProvider<?> p

@GetMapping(value = "/projects/{projectId}/jobs", produces = MediaType.APPLICATION_JSON_VALUE)
@ResponseStatus(HttpStatus.OK)
public Page<JobId> getJobs(@PathVariable String projectId,
@ParameterObject Pageable pageable,
@RequestParam(defaultValue = "") EnumSet<JobId.OptFields> optFields
public Page<Job> getJobs(@PathVariable String projectId,
@ParameterObject Pageable pageable,
@RequestParam(defaultValue = "") EnumSet<Job.OptFields> optFields
) {
return computeService.getJobs(projectsProvider.getProjectOrThrow(projectId), pageable, optFields);
}
Expand All @@ -90,9 +89,9 @@ public Page<JobId> getJobs(@PathVariable String projectId,
*/
@GetMapping(value = "/projects/{projectId}/jobs/{jobId}", produces = MediaType.APPLICATION_JSON_VALUE)
@ResponseStatus(HttpStatus.OK)
public JobId getJob(@PathVariable String projectId, @PathVariable String jobId,
@ParameterObject Pageable pageable,
@RequestParam(defaultValue = "progress") EnumSet<JobId.OptFields> optFields
public Job getJob(@PathVariable String projectId, @PathVariable String jobId,
@ParameterObject Pageable pageable,
@RequestParam(defaultValue = "progress") EnumSet<Job.OptFields> optFields
) {
return computeService.getJob(projectsProvider.getProjectOrThrow(projectId), jobId, optFields);
}
Expand All @@ -106,8 +105,8 @@ public JobId getJob(@PathVariable String projectId, @PathVariable String jobId,
*/
@PostMapping(value = "/projects/{projectId}/jobs", produces = MediaType.APPLICATION_JSON_VALUE)
@ResponseStatus(HttpStatus.ACCEPTED)
public JobId startJob(@PathVariable String projectId, @RequestBody JobSubmission jobSubmission,
@RequestParam(defaultValue = "command, progress") EnumSet<JobId.OptFields> optFields
public Job startJob(@PathVariable String projectId, @RequestBody JobSubmission jobSubmission,
@RequestParam(defaultValue = "command, progress") EnumSet<Job.OptFields> optFields
) {
return computeService.createAndSubmitJob(projectsProvider.getProjectOrThrow(projectId), jobSubmission, optFields);
}
Expand All @@ -124,9 +123,9 @@ public JobId startJob(@PathVariable String projectId, @RequestBody JobSubmission
*/
@PostMapping(value = "/projects/{projectId}/jobs/from-config", produces = MediaType.APPLICATION_JSON_VALUE)
@ResponseStatus(HttpStatus.ACCEPTED)
public JobId startJobFromConfig(@PathVariable String projectId, @RequestParam String jobConfigName, @RequestBody List<String> compoundIds,
@RequestParam(required = false) @Nullable Boolean recompute,
@RequestParam(defaultValue = "command, progress") EnumSet<JobId.OptFields> optFields
public Job startJobFromConfig(@PathVariable String projectId, @RequestParam String jobConfigName, @RequestBody List<String> compoundIds,
@RequestParam(required = false) @Nullable Boolean recompute,
@RequestParam(defaultValue = "command, progress") EnumSet<Job.OptFields> optFields
) {
final JobSubmission js = getJobConfig(jobConfigName, true);
js.setCompoundIds(compoundIds);
Expand All @@ -148,8 +147,8 @@ public JobId startJobFromConfig(@PathVariable String projectId, @RequestParam St
* @return JobId of background job that imports given run/compounds/features.
*/
@PostMapping(value = "/{projectId}/jobs/import-from-local-path", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
public JobId startImportFromPathJob(@PathVariable String projectId, @RequestBody ImportLocalFilesSubmission jobSubmission,
@RequestParam(defaultValue = "command, progress") EnumSet<JobId.OptFields> optFields
public Job startImportFromPathJob(@PathVariable String projectId, @RequestBody ImportLocalFilesSubmission jobSubmission,
@RequestParam(defaultValue = "command, progress") EnumSet<Job.OptFields> optFields
) throws IOException {
Project p = projectsProvider.getProjectOrThrow(projectId);
return computeService.createAndSubmitImportJob(p, jobSubmission, optFields);
Expand All @@ -165,8 +164,8 @@ public JobId startImportFromPathJob(@PathVariable String projectId, @RequestBody
* @return CompoundIds of the imported run/compounds/feature.
*/
@PostMapping(value = "/{projectId}/jobs/import-from-string", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
public JobId startImportFromStringJob(@PathVariable String projectId, @RequestBody ImportStringSubmission jobSubmission,
@RequestParam(defaultValue = "progress") EnumSet<JobId.OptFields> optFields
public Job startImportFromStringJob(@PathVariable String projectId, @RequestBody ImportStringSubmission jobSubmission,
@RequestParam(defaultValue = "progress") EnumSet<Job.OptFields> optFields
) throws IOException {
Project p = projectsProvider.getProjectOrThrow(projectId);
return computeService.createAndSubmitImportJob(p, jobSubmission, optFields);
Expand All @@ -189,7 +188,7 @@ public void deleteJob(@PathVariable String projectId,
@RequestParam(required = false, defaultValue = "true") boolean cancelIfRunning,
@RequestParam(required = false, defaultValue = "true") boolean awaitDeletion) {
computeService.deleteJob(projectsProvider.getProjectOrThrow(projectId), jobId, cancelIfRunning, awaitDeletion,
EnumSet.noneOf(JobId.OptFields.class));
EnumSet.noneOf(Job.OptFields.class));
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@

import de.unijena.bioinf.ms.frontend.subtools.InputFilesOptions;
import de.unijena.bioinf.ms.middleware.model.SearchQueryType;
import de.unijena.bioinf.ms.middleware.model.compute.JobId;
import de.unijena.bioinf.ms.middleware.model.compute.Job;
import de.unijena.bioinf.ms.middleware.model.projects.ProjectId;
import de.unijena.bioinf.ms.middleware.service.compute.ComputeService;
import de.unijena.bioinf.ms.middleware.service.projects.Project;
Expand Down Expand Up @@ -120,12 +120,12 @@ public ProjectId createProjectSpace(@PathVariable String projectId,
ProjectId pid = projectsProvider.createProjectSpace(projectId, Path.of(pathToProject));
Project project = projectsProvider.getProjectOrThrow(projectId);
if (inputFiles != null) {
JobId id = computeContext.createAndSubmitJob(project, List.of("project-space", "--keep-open"),
null, inputFiles, EnumSet.allOf(JobId.OptFields.class));
Job id = computeContext.createAndSubmitJob(project, List.of("project-space", "--keep-open"),
null, inputFiles, EnumSet.allOf(Job.OptFields.class));
if (awaitImport) { //todo maybe separate endpoint for non waiting.
try {
computeContext.getJJob(id.getId()).awaitResult();
computeContext.deleteJob(id.getId(), false, true, EnumSet.noneOf(JobId.OptFields.class));
computeContext.deleteJob(id.getId(), false, true, EnumSet.noneOf(Job.OptFields.class));
} catch (ExecutionException e) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Unexpected error when waiting for import jobs '" + id.getId() + "'.", e);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
@Getter
@Setter
@JsonInclude(JsonInclude.Include.NON_NULL)
public class JobId {
public class Job {
public enum OptFields {command, progress, affectedIds}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,5 +79,9 @@ public enum OptFields {topAnnotations, msData}
protected Annotations topAnnotations;
protected MsData msData;

protected boolean computing = false;
/**
* Wirte lock for this feature. If the feature is locked no write operations are possible.
* True if any computation is modifying this feature or its results
*/
protected boolean computing;
}
Loading

0 comments on commit 25dd696

Please sign in to comment.