Skip to content

Commit

Permalink
Merge pull request fli-iam#2206 from youennmerel/fli-iam#2194
Browse files Browse the repository at this point in the history
[fli-iam#2194] Generate Tags from "organ" dataset properties
  • Loading branch information
jcomedouteau authored Jun 13, 2024
2 parents e243c56 + 27b0a7f commit 3c9f281
Show file tree
Hide file tree
Showing 60 changed files with 1,021 additions and 298 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
CREATE TABLE `study_tag` (
`id` bigint(20) NOT NULL,
`color` varchar(255) DEFAULT NULL,
`name` varchar(255) DEFAULT NULL,
`study_id` bigint(20) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `FKboew1v3lqqa0afxnigq4fxhf3` (`study_id`),
CONSTRAINT `FKboew1v3lqqa0afxnigq4fxhf3` FOREIGN KEY (`study_id`) REFERENCES `study` (`id`)
);
CREATE TABLE `dataset_tag` (
`dataset_id` bigint(20) NOT NULL,
`study_tag_id` bigint(20) NOT NULL,
KEY `FKkh92b0ddi9nxrevqkdmvqpcm3` (`study_tag_id`),
KEY `FKd0dkfmqchgw18bxirml5an8ex` (`dataset_id`),
CONSTRAINT `FKd0dkfmqchgw18bxirml5an8ex` FOREIGN KEY (`dataset_id`) REFERENCES `dataset` (`id`),
CONSTRAINT `FKkh92b0ddi9nxrevqkdmvqpcm3` FOREIGN KEY (`study_tag_id`) REFERENCES `study_tag` (`id`)
);
48 changes: 48 additions & 0 deletions docker-compose/database/oneshot-updates/tag_ofsep_organs.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
-- Create study_tags in studies for all relevant studies

INSERT INTO studies.study_tag (color, name, study_id)
SELECT DISTINCT '#e74c3c', 'volume.organ:brain', proc.study_id
FROM datasets.dataset_property prop
INNER JOIN datasets.dataset_processing proc ON proc.id = prop.dataset_processing_id
WHERE prop.name = 'volume.organ' AND prop.value = 'brain';

INSERT INTO studies.study_tag (color, name, study_id)
SELECT DISTINCT '#27ae60', 'volume.organ:spine', proc.study_id
FROM datasets.dataset_property prop
INNER JOIN datasets.dataset_processing proc ON proc.id = prop.dataset_processing_id
WHERE prop.name = 'volume.organ' AND prop.value = 'spine';

INSERT INTO studies.study_tag (color, name, study_id)
SELECT DISTINCT '#198bda', 'volume.organ:null', proc.study_id
FROM datasets.dataset_property prop
INNER JOIN datasets.dataset_processing proc ON proc.id = prop.dataset_processing_id
WHERE prop.name = 'volume.organ' AND prop.value = 'null';

-- Copy studies study_tags to datasets study_tags

INSERT INTO datasets.study_tag (id, color, name, study_id)
SELECT id, color, name, study_id
FROM studies.study_tag;

-- Create dataset_tag for all relevant dataset

INSERT INTO dataset_tag (dataset_id, study_tag_id)
SELECT DISTINCT prop.dataset_id, tag.id
FROM dataset_property prop
INNER JOIN dataset_processing proc ON proc.id = prop.dataset_processing_id
INNER JOIN study_tag tag ON tag.study_id = proc.study_id AND tag.name = 'volume.organ:brain'
WHERE prop.name = 'volume.organ' AND prop.value = 'brain';

INSERT INTO dataset_tag (dataset_id, study_tag_id)
SELECT DISTINCT prop.dataset_id, tag.id
FROM dataset_property prop
INNER JOIN dataset_processing proc ON proc.id = prop.dataset_processing_id
INNER JOIN study_tag tag ON tag.study_id = proc.study_id AND tag.name = 'volume.organ:spine'
WHERE prop.name = 'volume.organ' AND prop.value = 'spine';

INSERT INTO dataset_tag (dataset_id, study_tag_id)
SELECT DISTINCT prop.dataset_id, tag.id
FROM dataset_property prop
INNER JOIN dataset_processing proc ON proc.id = prop.dataset_processing_id
INNER JOIN study_tag tag ON tag.study_id = proc.study_id AND tag.name = 'volume.organ:null'
WHERE prop.name = 'volume.organ' AND prop.value = 'null';
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,9 @@

package org.shanoir.ng.configuration.amqp;

import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.persistence.EntityManager;
import org.apache.solr.client.solrj.SolrServerException;
import org.shanoir.ng.bids.service.BIDSService;
import org.shanoir.ng.dataset.dto.StudyStorageVolumeDTO;
Expand All @@ -36,17 +35,9 @@
import org.shanoir.ng.shared.event.ShanoirEvent;
import org.shanoir.ng.shared.event.ShanoirEventService;
import org.shanoir.ng.shared.event.ShanoirEventType;
import org.shanoir.ng.shared.model.AcquisitionEquipment;
import org.shanoir.ng.shared.model.Center;
import org.shanoir.ng.shared.model.Study;
import org.shanoir.ng.shared.model.Subject;
import org.shanoir.ng.shared.model.SubjectStudy;
import org.shanoir.ng.shared.model.Tag;
import org.shanoir.ng.shared.repository.AcquisitionEquipmentRepository;
import org.shanoir.ng.shared.repository.CenterRepository;
import org.shanoir.ng.shared.repository.StudyRepository;
import org.shanoir.ng.shared.repository.SubjectRepository;
import org.shanoir.ng.shared.repository.SubjectStudyRepository;
import org.shanoir.ng.shared.model.*;
import org.shanoir.ng.shared.repository.*;
import org.shanoir.ng.shared.service.StudyService;
import org.shanoir.ng.shared.subjectstudy.SubjectStudyDTO;
import org.shanoir.ng.shared.subjectstudy.SubjectType;
import org.shanoir.ng.solr.service.SolrService;
Expand All @@ -58,11 +49,8 @@
import org.slf4j.LoggerFactory;
import org.springframework.amqp.AmqpRejectAndDontRequeueException;
import org.springframework.amqp.core.ExchangeTypes;
import org.springframework.amqp.rabbit.annotation.Exchange;
import org.springframework.amqp.rabbit.annotation.Queue;
import org.springframework.amqp.rabbit.annotation.QueueBinding;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.amqp.rabbit.annotation.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.repository.CrudRepository;
import org.springframework.scheduling.annotation.Async;
Expand All @@ -72,10 +60,9 @@
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;

import jakarta.persistence.EntityManager;
import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;

/**
* RabbitMQ configuration.
Expand All @@ -87,6 +74,7 @@ public class RabbitMQDatasetsService {

@Autowired
private DatasetService datasetService;

@Autowired
private DatasetCopyService datasetCopyService;

Expand Down Expand Up @@ -118,7 +106,7 @@ public class RabbitMQDatasetsService {
private ExaminationService examinationService;

@Autowired
ShanoirEventService eventService;
public ShanoirEventService eventService;

@Autowired
private ExaminationRepository examinationRepository;
Expand All @@ -134,6 +122,10 @@ public class RabbitMQDatasetsService {

@Autowired
private ObjectMapper objectMapper;

@Autowired
private StudyService studyService;

@Autowired
EntityManager entityManager;

Expand Down Expand Up @@ -175,62 +167,39 @@ private SubjectStudy dtoToSubjectStudy(SubjectStudyDTO dto) {
return subjectStudy;
}

@Transactional
@RabbitListener(queues = RabbitMQConfiguration.STUDY_NAME_UPDATE_QUEUE, containerFactory = "singleConsumerFactory")
@RabbitHandler
public void receiveStudyNameUpdate(final String studyStr) {
public String receiveStudyUpdate(final String studyAsString) {
try {
Study received = objectMapper.readValue(studyStr, Study.class);
bidsService.deleteBidsFolder(received.getId(), null);
Study stud = receiveAndUpdateIdNameEntity(studyStr, Study.class, studyRepository);

// TAGS
if (stud.getTags() != null) {
stud.getTags().clear();
} else {
stud.setTags(new ArrayList<>());
}
if (received.getTags() != null) {
stud.getTags().addAll(received.getTags());
}
for (Tag tag : stud.getTags()) {
tag.setStudy(stud);
}
if (stud.getId() == null) throw new IllegalStateException("The entity should must have an id ! Received string : \"" + studyStr + "\"");
Study studyDb = this.studyRepository.save(stud);
Study updated = objectMapper.readValue(studyAsString, Study.class);

// SUBJECT_STUDY
if (stud.getSubjectStudyList() != null) {
stud.getSubjectStudyList().clear();
} else {
stud.setSubjectStudyList(new ArrayList<>());
}
if (received.getSubjectStudyList() != null) {
stud.getSubjectStudyList().addAll(received.getSubjectStudyList());
}
for (SubjectStudy sustu : stud.getSubjectStudyList()) {
sustu.setStudy(stud);
for (Tag tag : sustu.getTags()) {
if (tag.getId() == null) {
Tag dbTag = studyDb.getTags().stream().filter(upTag ->
upTag.getColor().equals(tag.getColor()) && upTag.getName().equals(tag.getName())
).findFirst().orElse(null);
if (dbTag != null) {
tag.setId(dbTag.getId());
} else {
throw new IllegalStateException("Cannot link a new tag to a subject-study, this tag does not exist in the study");
}
}
}
bidsService.deleteBidsFolder(updated.getId(), null);

Study current = this.receiveAndUpdateIdNameEntity(studyAsString, Study.class, studyRepository);

List<String> errors = studyService.validate(updated, current);

if(!errors.isEmpty()){
return errors.get(0);
}
if (stud.getId() == null) throw new IllegalStateException("The entity should must have an id ! Received string : \"" + studyStr + "\"");
this.studyRepository.save(stud);
List<Long> subjectIds = new ArrayList<>();
stud.getSubjectStudyList().forEach(subStu -> subjectIds.add(subStu.getSubject().getId()));
updateSolr(subjectIds);
} catch (Exception e) {
throw new AmqpRejectAndDontRequeueException(RABBIT_MQ_ERROR, e);

studyService.updateStudy(updated, current);

List<Long> subjectIds = current.getSubjectStudyList()
.stream().map(subStu ->
subStu.getSubject().getId()
).collect(Collectors.toList()
);

solrService.updateSubjects(subjectIds);

} catch (Exception ex) {
LOG.error("An error occured while processing study update", ex);
return ex.getMessage();
}

return "";
}

@Transactional
Expand Down Expand Up @@ -260,7 +229,7 @@ public boolean receiveSubjectNameUpdate(final String subjectStr) {
// Update solr references
List<Long> subjectIdList = new ArrayList<Long>();
subjectIdList.add(su.getId());
updateSolr(subjectIdList);
solrService.updateSubjects(subjectIdList);

// Update BIDS
Set<Long> studyIds = new HashSet<>();
Expand All @@ -277,24 +246,6 @@ public boolean receiveSubjectNameUpdate(final String subjectStr) {
}
}

/**
* Updates all the solr references for this subject.
* @param subjectIds the subject ID updated
*/
private void updateSolr(final List<Long> subjectIds) throws SolrServerException, IOException {
Set<Long> datasetsToUpdate = new HashSet<>();
for (Examination exam : examinationRepository.findBySubjectIdIn(subjectIds)) {
for (DatasetAcquisition acq : exam.getDatasetAcquisitions()) {
for (Dataset ds : acq.getDatasets()) {
datasetsToUpdate.add(ds.getId());
}
}
}
if (!CollectionUtils.isEmpty(datasetsToUpdate)) {
this.solrService.indexDatasets(new ArrayList<>(datasetsToUpdate));
}
}

@Transactional
@RabbitListener(queues = RabbitMQConfiguration.ACQUISITION_EQUIPEMENT_UPDATE_QUEUE, containerFactory = "singleConsumerFactory")
@RabbitHandler
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import jakarta.mail.MessagingException;
import jakarta.servlet.http.HttpServletResponse;
import jakarta.validation.Valid;
import org.apache.solr.client.solrj.SolrServerException;
import org.shanoir.ng.dataset.dto.DatasetAndProcessingsDTOInterface;
import org.shanoir.ng.dataset.dto.DatasetDTO;
import org.shanoir.ng.dataset.model.Dataset;
Expand Down Expand Up @@ -325,4 +326,19 @@ ResponseEntity<ByteArrayResource> downloadStatistics(
ResponseEntity<List<DatasetAndProcessingsDTOInterface>> findDatasetsByIds(
@RequestParam(value = "datasetIds", required = true) List<Long> datasetIds);

@Operation(summary = "", description = "Updates the study tags of a dataset")
@ApiResponses(value = { @ApiResponse(responseCode = "204", description = "dataset study tags updated"),
@ApiResponse(responseCode = "401", description = "unauthorized"),
@ApiResponse(responseCode = "403", description = "forbidden"),
@ApiResponse(responseCode = "404", description = "dataset does not exists"),
@ApiResponse(responseCode = "422", description = "bad parameters"),
@ApiResponse(responseCode = "500", description = "unexpected error") })
@PutMapping(value = "/{datasetId}/tags", produces = { "application/json" }, consumes = {
"application/json" })
@PreAuthorize("hasRole('ADMIN') or (hasRole('EXPERT') and @datasetSecurityService.hasUpdateRightOnDataset(#dataset, 'CAN_ADMINISTRATE'))")
ResponseEntity<Void> updateDatasetTags(
@Parameter(description = "id of the dataset", required = true) @PathVariable("datasetId") Long datasetId,
@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "array of study tag ids", required = true) @RequestBody List<Long> studyTagIds,
BindingResult result) throws RestServiceException, EntityNotFoundException, SolrServerException, IOException;

}
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import org.shanoir.ng.dataset.modality.MrDatasetMapper;
import org.shanoir.ng.dataset.model.Dataset;
import org.shanoir.ng.dataset.model.DatasetExpressionFormat;
import org.shanoir.ng.dataset.repository.DatasetRepository;
import org.shanoir.ng.dataset.service.DatasetDownloaderServiceImpl;
import org.shanoir.ng.dataset.service.DatasetService;
import org.shanoir.ng.datasetacquisition.model.DatasetAcquisition;
Expand All @@ -55,13 +56,12 @@
import org.shanoir.ng.importer.service.ImporterService;
import org.shanoir.ng.shared.configuration.RabbitMQConfiguration;
import org.shanoir.ng.shared.error.FieldErrorMap;
import org.shanoir.ng.shared.event.ShanoirEvent;
import org.shanoir.ng.shared.event.ShanoirEventService;
import org.shanoir.ng.shared.event.ShanoirEventType;
import org.shanoir.ng.shared.exception.*;
import org.shanoir.ng.solr.service.SolrService;
import org.shanoir.ng.tag.model.StudyTag;
import org.shanoir.ng.tag.service.StudyTagService;
import org.shanoir.ng.utils.DatasetFileUtils;
import org.shanoir.ng.utils.KeycloakUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
Expand Down Expand Up @@ -135,6 +135,12 @@ public class DatasetApiController implements DatasetApi {
@Autowired
private ObjectMapper objectMapper;

@Autowired
private StudyTagService studyTagService;

@Autowired
private DatasetRepository datasetRepository;

/** Number of downloadable datasets. */
private static final int DATASET_LIMIT = 500;

Expand Down Expand Up @@ -243,6 +249,23 @@ public ResponseEntity<List<DatasetAndProcessingsDTOInterface>> findDatasetsByIds
}

@Override
public ResponseEntity<Void> updateDatasetTags(Long datasetId, List<Long> studyTagIds, BindingResult result) throws EntityNotFoundException, SolrServerException, IOException {
Dataset ds = datasetService.findById(datasetId);
if (ds == null) {
throw new EntityNotFoundException(Dataset.class, datasetId);
}

List<StudyTag> tags = studyTagService.findByIds(studyTagIds);
ds.setTags(tags);

datasetRepository.save(ds);

solrService.indexDataset(datasetId);

return new ResponseEntity<>(HttpStatus.OK);
}

@Override
public ResponseEntity<List<DatasetDTO>> findDatasetsByExaminationId(Long examinationId) {
List<Dataset> datasets = datasetService.findByExaminationId(examinationId);
if (datasets.isEmpty()) {
Expand Down
Loading

0 comments on commit 3c9f281

Please sign in to comment.