Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Do not use clinical data MVs #10847

Merged
merged 8 commits into from
Jun 24, 2024
Merged

Do not use clinical data MVs #10847

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
2616360
use clinical_data_derived instead of sample_clinical_attribute_numeri…
onursumer Jun 20, 2024
a835a46
use clinical_attribute_meta instead of sample_clinical_attribute_nume…
onursumer Jun 20, 2024
46ab2c5
remove unused clinical data count methods and SQL
onursumer Jun 21, 2024
4aae411
fix numericalClinicalDataCountFilter
onursumer Jun 21, 2024
35dad1e
Move CategoricalClinicalAttributeFilter to repository
haynescd Jun 24, 2024
8fd08c2
remove unused columns
haynescd Jun 24, 2024
8515fa9
Merge remote-tracking branch 'origin/demo-rfc80-poc' into do-not-use-…
haynescd Jun 24, 2024
29d7105
Add override to methods
haynescd Jun 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 14 additions & 19 deletions src/main/java/org/cbioportal/persistence/StudyViewRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
package org.cbioportal.persistence;

import org.cbioportal.model.AlterationCountByGene;
import org.cbioportal.model.ClinicalAttribute;
import org.cbioportal.model.ClinicalData;
import org.cbioportal.model.ClinicalDataCount;
import org.cbioportal.model.CopyNumberCountByGene;
import org.cbioportal.model.GenomicDataCount;
import org.cbioportal.model.Sample;
import org.cbioportal.persistence.enums.ClinicalAttributeDataSource;
import org.cbioportal.persistence.enums.ClinicalAttributeDataType;
import org.cbioportal.web.parameter.CategorizedClinicalDataCountFilter;
import org.cbioportal.web.parameter.StudyViewFilter;

Expand All @@ -16,32 +15,28 @@
import java.util.Set;

public interface StudyViewRepository {
List<Sample> getFilteredSamples(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter);
List<Sample> getFilteredSamples(StudyViewFilter studyViewFilter);

List<ClinicalData> getSampleClinicalData(StudyViewFilter studyViewFilter, List<String> attributeIds, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter);
List<ClinicalData> getSampleClinicalData(StudyViewFilter studyViewFilter, List<String> attributeIds);

List<ClinicalData> getPatientClinicalData(StudyViewFilter studyViewFilter, List<String> attributeIds, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter);
List<ClinicalData> getPatientClinicalData(StudyViewFilter studyViewFilter, List<String> attributeIds);

List<AlterationCountByGene> getMutatedGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter);
List<AlterationCountByGene> getMutatedGenes(StudyViewFilter studyViewFilter);

List<CopyNumberCountByGene> getCnaGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter);
List<AlterationCountByGene> getStructuralVariantGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter);
List<AlterationCountByGene> getStructuralVariantGenes(StudyViewFilter studyViewFilter);
List<CopyNumberCountByGene> getCnaGenes(StudyViewFilter studyViewFilter);

List<ClinicalDataCount> getClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, List<String> filteredAttributes);
List<ClinicalDataCount> getClinicalDataCounts(StudyViewFilter studyViewFilter, List<String> filteredAttributes);

List<ClinicalDataCount> getSampleClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, List<String> filteredAttributes);
List<GenomicDataCount> getGenomicDataCounts(StudyViewFilter studyViewFilter);

List<ClinicalDataCount> getPatientClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, List<String> filteredAttributes);

List<GenomicDataCount> getGenomicDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter);

List<String> getClinicalDataAttributeNames(ClinicalAttributeDataSource clinicalAttributeDataSource, ClinicalAttributeDataType dataType);
List<ClinicalAttribute> getClinicalAttributes();

Map<String, AlterationCountByGene> getTotalProfiledCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, String alterationType);
Map<String, AlterationCountByGene> getTotalProfiledCounts(StudyViewFilter studyViewFilter, String alterationType);

int getFilteredSamplesCount(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter);
int getFilteredSamplesCount(StudyViewFilter studyViewFilter);

Map<String, Set<String>> getMatchingGenePanelIds(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, String alterationType);
Map<String, Set<String>> getMatchingGenePanelIds(StudyViewFilter studyViewFilter, String alterationType);

int getTotalProfiledCountsByAlterationType(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, String alterationType);
int getTotalProfiledCountsByAlterationType(StudyViewFilter studyViewFilter, String alterationType);
}
2 changes: 1 addition & 1 deletion src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ List<ClinicalDataCount> getSampleClinicalDataCounts(StudyViewFilter studyViewFil
List<ClinicalDataCount> getClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter,
boolean applyPatientIdFilters, List<String> attributeIds, List<String> filteredAttributeValues);

List<String> getClinicalAttributeNames(String tableName);
List<ClinicalAttribute> getClinicalAttributes();

List<ClinicalData> getSampleClinicalDataFromStudyViewFilter(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, boolean applyPatientIdFilters, List<String> attributeIds);

Expand Down
116 changes: 83 additions & 33 deletions src/main/java/org/cbioportal/persistence/mybatisclickhouse/StudyViewMyBatisRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.cbioportal.persistence.mybatisclickhouse;
import org.cbioportal.model.AlterationCountByGene;
import org.cbioportal.model.ClinicalAttribute;
import org.cbioportal.model.ClinicalData;
import org.cbioportal.model.ClinicalDataCount;
import org.cbioportal.model.GenePanelToGene;
Expand All @@ -8,13 +9,13 @@
import org.cbioportal.model.Sample;
import org.cbioportal.persistence.StudyViewRepository;
import org.cbioportal.persistence.enums.ClinicalAttributeDataSource;
import org.cbioportal.persistence.enums.ClinicalAttributeDataType;
import org.cbioportal.persistence.helper.AlterationFilterHelper;
import org.cbioportal.web.parameter.CategorizedClinicalDataCountFilter;
import org.cbioportal.web.parameter.StudyViewFilter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Repository;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand All @@ -23,6 +24,9 @@
@Repository
public class StudyViewMyBatisRepository implements StudyViewRepository {

private Map<ClinicalAttributeDataSource, List<ClinicalAttribute>> clinicalAttributesMap = new HashMap<>();


private static final List<String> FILTERED_CLINICAL_ATTR_VALUES = Collections.emptyList();
private final StudyViewMapper mapper;

Expand All @@ -31,89 +35,82 @@ public StudyViewMyBatisRepository(StudyViewMapper mapper) {
this.mapper = mapper;
}
@Override
public List<Sample> getFilteredSamples(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) {

public List<Sample> getFilteredSamples(StudyViewFilter studyViewFilter) {
CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter);
return mapper.getFilteredSamples(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter));
}

@Override
public List<AlterationCountByGene> getMutatedGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) {
public List<AlterationCountByGene> getMutatedGenes(StudyViewFilter studyViewFilter) {
CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter);
return mapper.getMutatedGenes(studyViewFilter, categorizedClinicalDataCountFilter,
shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter),
AlterationFilterHelper.build(studyViewFilter.getAlterationFilter()));
}

@Override
public List<CopyNumberCountByGene> getCnaGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) {
public List<CopyNumberCountByGene> getCnaGenes(StudyViewFilter studyViewFilter) {
CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter);
return mapper.getCnaGenes(studyViewFilter, categorizedClinicalDataCountFilter,
shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter),
AlterationFilterHelper.build(studyViewFilter.getAlterationFilter()));
}

@Override
public List<AlterationCountByGene> getStructuralVariantGenes(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) {
public List<AlterationCountByGene> getStructuralVariantGenes(StudyViewFilter studyViewFilter) {
CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter);
return mapper.getStructuralVariantGenes(studyViewFilter, categorizedClinicalDataCountFilter,
shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter),
AlterationFilterHelper.build(studyViewFilter.getAlterationFilter()));
}

@Override
public List<ClinicalDataCount> getClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, List<String> filteredAttributes) {
public List<ClinicalDataCount> getClinicalDataCounts(StudyViewFilter studyViewFilter, List<String> filteredAttributes) {
CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter);
return mapper.getClinicalDataCounts(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter),
filteredAttributes, FILTERED_CLINICAL_ATTR_VALUES );
}

@Override
public List<GenomicDataCount> getGenomicDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) {
public List<GenomicDataCount> getGenomicDataCounts(StudyViewFilter studyViewFilter) {
CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter);
return mapper.getGenomicDataCounts(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter));
}



@Override
public List<ClinicalDataCount> getSampleClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, List<String> filteredAttributes) {
return mapper.getSampleClinicalDataCounts(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter),
filteredAttributes, FILTERED_CLINICAL_ATTR_VALUES );
}

@Override
public List<ClinicalDataCount> getPatientClinicalDataCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, List<String> filteredAttributes) {
return mapper.getPatientClinicalDataCounts(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter),
filteredAttributes, FILTERED_CLINICAL_ATTR_VALUES);
}

@Override
public List<String> getClinicalDataAttributeNames(ClinicalAttributeDataSource clinicalAttributeDataSource, ClinicalAttributeDataType dataType) {
String tableName = clinicalAttributeDataSource.getValue().toLowerCase() + "_clinical_attribute_" + dataType.getValue().toLowerCase() + "_mv";
return mapper.getClinicalAttributeNames(tableName);
public List<ClinicalAttribute> getClinicalAttributes() {
return mapper.getClinicalAttributes();
}

private boolean shouldApplyPatientIdFilters(CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) {
return categorizedClinicalDataCountFilter.getPatientCategoricalClinicalDataFilters() != null && !categorizedClinicalDataCountFilter.getPatientCategoricalClinicalDataFilters().isEmpty()
|| categorizedClinicalDataCountFilter.getPatientNumericalClinicalDataFilters() != null && !categorizedClinicalDataCountFilter.getPatientNumericalClinicalDataFilters().isEmpty();
}

public List<ClinicalData> getSampleClinicalData(StudyViewFilter studyViewFilter, List<String> attributeIds, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) {
public List<ClinicalData> getSampleClinicalData(StudyViewFilter studyViewFilter, List<String> attributeIds) {
CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter);
return mapper.getSampleClinicalDataFromStudyViewFilter(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter), attributeIds);
}

public List<ClinicalData> getPatientClinicalData(StudyViewFilter studyViewFilter, List<String> attributeIds, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) {
public List<ClinicalData> getPatientClinicalData(StudyViewFilter studyViewFilter, List<String> attributeIds) {
CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter);
return mapper.getPatientClinicalDataFromStudyViewFilter(studyViewFilter, categorizedClinicalDataCountFilter, shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter), attributeIds);
}

public Map<String, AlterationCountByGene> getTotalProfiledCounts(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, String alterationType) {
public Map<String, AlterationCountByGene> getTotalProfiledCounts(StudyViewFilter studyViewFilter, String alterationType) {
CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter);
return mapper.getTotalProfiledCounts(studyViewFilter, categorizedClinicalDataCountFilter,
shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter), alterationType);
}

@Override
public int getFilteredSamplesCount(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter) {
public int getFilteredSamplesCount(StudyViewFilter studyViewFilter) {
CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter);
return mapper.getFilteredSamplesCount(studyViewFilter, categorizedClinicalDataCountFilter,
shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter));
}

@Override
public Map<String, Set<String>> getMatchingGenePanelIds(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, String alterationType) {
public Map<String, Set<String>> getMatchingGenePanelIds(StudyViewFilter studyViewFilter, String alterationType) {
CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter);
return mapper.getMatchingGenePanelIds(studyViewFilter, categorizedClinicalDataCountFilter,
shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter), alterationType)
.stream()
Expand All @@ -122,9 +119,62 @@ public Map<String, Set<String>> getMatchingGenePanelIds(StudyViewFilter studyVie
}

@Override
public int getTotalProfiledCountsByAlterationType(StudyViewFilter studyViewFilter, CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter, String alterationType) {
public int getTotalProfiledCountsByAlterationType(StudyViewFilter studyViewFilter, String alterationType) {
CategorizedClinicalDataCountFilter categorizedClinicalDataCountFilter = extractClinicalDataCountFilters(studyViewFilter);
return mapper.getTotalProfiledCountByAlterationType(studyViewFilter, categorizedClinicalDataCountFilter,
shouldApplyPatientIdFilters(categorizedClinicalDataCountFilter), alterationType);
}

private void buildClinicalAttributeNameMap() {
clinicalAttributesMap = this.getClinicalAttributes()
.stream()
.collect(Collectors.groupingBy(ca -> ca.getPatientAttribute() ? ClinicalAttributeDataSource.PATIENT : ClinicalAttributeDataSource.SAMPLE));
}

private CategorizedClinicalDataCountFilter extractClinicalDataCountFilters(final StudyViewFilter studyViewFilter) {
if (clinicalAttributesMap.isEmpty()) {
buildClinicalAttributeNameMap();
}

if (studyViewFilter.getClinicalDataFilters() == null) {
return CategorizedClinicalDataCountFilter.getBuilder().build();
}

List<String> patientCategoricalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.PATIENT)
.stream().filter(ca -> ca.getDatatype().equals("STRING"))
.map(ClinicalAttribute::getAttrId)
.toList();

List<String> patientNumericalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.PATIENT)
.stream().filter(ca -> ca.getDatatype().equals("NUMBER"))
.map(ClinicalAttribute::getAttrId)
.toList();

List<String> sampleCategoricalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.SAMPLE)
.stream().filter(ca -> ca.getDatatype().equals("STRING"))
.map(ClinicalAttribute::getAttrId)
.toList();

List<String> sampleNumericalAttributes = clinicalAttributesMap.get(ClinicalAttributeDataSource.SAMPLE)
.stream().filter(ca -> ca.getDatatype().equals("NUMBER"))
.map(ClinicalAttribute::getAttrId)
.toList();

return CategorizedClinicalDataCountFilter.getBuilder()
.setPatientCategoricalClinicalDataFilters(studyViewFilter.getClinicalDataFilters()
.stream().filter(clinicalDataFilter -> patientCategoricalAttributes.contains(clinicalDataFilter.getAttributeId()))
.collect(Collectors.toList()))
.setPatientNumericalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream()
.filter(clinicalDataFilter -> patientNumericalAttributes.contains(clinicalDataFilter.getAttributeId()))
.collect(Collectors.toList()))
.setSampleCategoricalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream()
.filter(clinicalDataFilter -> sampleCategoricalAttributes.contains(clinicalDataFilter.getAttributeId()))
.collect(Collectors.toList()))
.setSampleNumericalClinicalDataFilters(studyViewFilter.getClinicalDataFilters().stream()
.filter(clinicalDataFilter -> sampleNumericalAttributes.contains(clinicalDataFilter.getAttributeId()))
.collect(Collectors.toList()))
.build();
}


}
Loading