diff --git a/cumulus/loaders/i2b2/loader.py b/cumulus/loaders/i2b2/loader.py index 848ddfbf..d56596ac 100644 --- a/cumulus/loaders/i2b2/loader.py +++ b/cumulus/loaders/i2b2/loader.py @@ -6,8 +6,6 @@ from functools import partial from typing import Callable, Iterable, List, TypeVar -from fhirclient.models.resource import Resource - from cumulus import store from cumulus.loaders.base import Loader from cumulus.loaders.i2b2 import extract, schema, transform @@ -16,7 +14,7 @@ AnyDimension = TypeVar("AnyDimension", bound=schema.Dimension) I2b2ExtractorCallable = Callable[[], Iterable[schema.Dimension]] CsvToI2b2Callable = Callable[[str], Iterable[schema.Dimension]] -I2b2ToFhirCallable = Callable[[AnyDimension], Resource] +I2b2ToFhirCallable = Callable[[AnyDimension], dict] class I2b2Loader(Loader): @@ -110,10 +108,10 @@ def _loop(self, i2b2_entries: Iterable[schema.Dimension], to_fhir: I2b2ToFhirCal # Now write each FHIR resource line by line to the output # (we do this all line by line via generators to avoid loading everything in memory at once) for resource in fhir_resources: - if resource.id in ids: + if resource["id"] in ids: continue - ids.add(resource.id) - json.dump(resource.as_json(), output_file) + ids.add(resource["id"]) + json.dump(resource, output_file) output_file.write("\n") ################################################################################################################### diff --git a/cumulus/loaders/i2b2/transform.py b/cumulus/loaders/i2b2/transform.py index 08c3eb51..137894c2 100644 --- a/cumulus/loaders/i2b2/transform.py +++ b/cumulus/loaders/i2b2/transform.py @@ -4,22 +4,6 @@ import logging from typing import Optional -from fhirclient.models.address import Address -from fhirclient.models.attachment import Attachment -from fhirclient.models.codeableconcept import CodeableConcept -from fhirclient.models.coding import Coding -from fhirclient.models.condition import Condition -from fhirclient.models.documentreference import DocumentReference, DocumentReferenceContext, DocumentReferenceContent -from fhirclient.models.duration import Duration -from fhirclient.models.encounter import Encounter -from fhirclient.models.extension import Extension -from fhirclient.models.fhirdate import FHIRDate -from fhirclient.models.fhirreference import FHIRReference -from fhirclient.models.meta import Meta -from fhirclient.models.observation import Observation -from fhirclient.models.patient import Patient -from fhirclient.models.period import Period - from cumulus import fhir_common from cumulus.loaders.i2b2 import external_mappings from cumulus.loaders.i2b2.schema import PatientDimension, VisitDimension, ObservationFact @@ -32,132 +16,117 @@ ############################################################################### -def to_fhir_patient(patient: PatientDimension) -> Patient: +def to_fhir_patient(patient: PatientDimension) -> dict: """ :param patient: i2b2 Patient Dimension record :return: https://www.hl7.org/fhir/patient.html """ - subject = Patient() - subject.meta = Meta({"profile": ["http://hl7.org/fhir/us/core/StructureDefinition/us-core-patient"]}) - subject.id = patient.patient_num + subject = { + "resourceType": "Patient", + "id": str(patient.patient_num), + "meta": {"profile": ["http://hl7.org/fhir/us/core/StructureDefinition/us-core-patient"]}, + } if patient.birth_date: - subject.birthDate = parse_fhir_date(patient.birth_date) + subject["birthDate"] = chop_to_date(patient.birth_date) if patient.death_date: - subject.deceasedDateTime = parse_fhir_date(patient.death_date) + subject["deceasedDateTime"] = chop_to_date(patient.death_date) if patient.sex_cd: - subject.gender = parse_gender(patient.sex_cd) + subject["gender"] = external_mappings.FHIR_GENDER.get(patient.sex_cd, "other") # TODO: verify that i2b2 always has a single patient address, always in US if patient.zip_cd: - subject.address = [Address({"country": "US", "postalCode": parse_zip_code(patient.zip_cd)})] + subject["address"] = [{"country": "US", "postalCode": patient.zip_cd}] if patient.race_cd: # race_cd can be either a race or an ethnicity. In FHIR, those are two different extensions. race_code = external_mappings.CDC_RACE.get(patient.race_cd) if race_code is not None: - subject.extension = [ - Extension( - { - "url": "http://hl7.org/fhir/us/core/StructureDefinition/us-core-race", - "extension": [ - { - "url": "ombCategory", - "valueCoding": { - "system": race_code[0], - "code": race_code[1], - "display": patient.race_cd, - }, + subject["extension"] = [ + { + "url": "http://hl7.org/fhir/us/core/StructureDefinition/us-core-race", + "extension": [ + { + "url": "ombCategory", + "valueCoding": { + "system": race_code[0], + "code": race_code[1], + "display": patient.race_cd, }, - ], - } - ) + }, + ], + }, ] ethnicity_code = external_mappings.CDC_ETHNICITY.get(patient.race_cd) if ethnicity_code is not None: - subject.extension = [ - Extension( - { - "url": "http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity", - "extension": [ - { - "url": "ombCategory", - "valueCoding": { - "system": ethnicity_code[0], - "code": ethnicity_code[1], - "display": patient.race_cd, - }, + subject["extension"] = [ + { + "url": "http://hl7.org/fhir/us/core/StructureDefinition/us-core-ethnicity", + "extension": [ + { + "url": "ombCategory", + "valueCoding": { + "system": ethnicity_code[0], + "code": ethnicity_code[1], + "display": patient.race_cd, }, - ], - } - ) + }, + ], + }, ] + return subject -def to_fhir_encounter(visit: VisitDimension) -> Encounter: +def to_fhir_encounter(visit: VisitDimension) -> dict: """ :param visit: i2b2 Visit Dimension Record :return: https://www.hl7.org/fhir/encounter.html """ - encounter = Encounter() - encounter.meta = Meta({"profile": ["http://hl7.org/fhir/us/core/StructureDefinition/us-core-encounter"]}) - encounter.id = str(visit.encounter_num) - encounter.subject = ref_subject(visit.patient_num) - encounter.status = "unknown" - - # Most generic encounter type possible, only included because the 'type' field is required in us-core - encounter.type = [make_concept("308335008", "http://snomed.info/sct", "Patient encounter procedure")] - - encounter.period = Period( - { - "start": parse_fhir_date_isostring(visit.start_date), - "end": parse_fhir_date_isostring(visit.end_date), - } - ) + encounter = { + "resourceType": "Encounter", + "id": str(visit.encounter_num), + "subject": fhir_common.ref_resource("Patient", visit.patient_num), + "meta": {"profile": ["http://hl7.org/fhir/us/core/StructureDefinition/us-core-encounter"]}, + "status": "unknown", + "period": {"start": chop_to_date(visit.start_date), "end": chop_to_date(visit.end_date)}, + # Most generic encounter type possible, only included because the 'type' field is required in us-core + "type": [make_concept("308335008", "http://snomed.info/sct", "Patient encounter procedure")], + } if visit.length_of_stay: # days - encounter.length = Duration({"unit": "d", "value": parse_fhir_duration(visit.length_of_stay)}) + encounter["length"] = {"unit": "d", "value": visit.length_of_stay and float(visit.length_of_stay)} class_fhir = external_mappings.SNOMED_ADMISSION.get(visit.inout_cd) if not class_fhir: logging.debug("unknown encounter.class_fhir.code for i2b2 INOUT_CD : %s", visit.inout_cd) class_fhir = "?" # bogus value, but FHIR demands *some* class value - encounter.class_fhir = Coding( - { - "system": "http://terminology.hl7.org/CodeSystem/v3-ActCode", - "code": class_fhir, - } - ) + encounter["class"] = { + "system": "http://terminology.hl7.org/CodeSystem/v3-ActCode", + "code": class_fhir, + } return encounter -def to_fhir_observation(obsfact: ObservationFact) -> Observation: - """ - :param obsfact: base "FHIR Observation" from base "I2B2 ObservationFact" - :return: https://www.hl7.org/fhir/observation.html - """ - observation = Observation() - observation.id = obsfact.instance_num - observation.subject = ref_subject(obsfact.patient_num) - observation.encounter = ref_encounter(obsfact.encounter_num) - observation.effectiveDateTime = parse_fhir_date(obsfact.start_date) - - return observation - - -def to_fhir_observation_lab(obsfact: ObservationFact) -> Observation: +def to_fhir_observation_lab(obsfact: ObservationFact) -> dict: """ :param obsfact: i2b2 observation fact containing the LAB NAME AND VALUE :return: https://www.hl7.org/fhir/observation.html """ - observation = to_fhir_observation(obsfact) - observation.status = "unknown" + observation = { + "resourceType": "Observation", + "id": str(obsfact.instance_num), + "subject": fhir_common.ref_resource("Patient", obsfact.patient_num), + "encounter": fhir_common.ref_resource("Encounter", obsfact.encounter_num), + "category": [make_concept("laboratory", "http://terminology.hl7.org/CodeSystem/observation-category")], + "effectiveDateTime": chop_to_date(obsfact.start_date), + "status": "unknown", + } if obsfact.concept_cd in external_mappings.LOINC_COVID_LAB_TESTS: obs_code = external_mappings.LOINC_COVID_LAB_TESTS[obsfact.concept_cd] @@ -165,58 +134,55 @@ def to_fhir_observation_lab(obsfact: ObservationFact) -> Observation: else: obs_code = obsfact.concept_cd obs_system = "http://cumulus.smarthealthit.org/i2b2" - - observation.code = make_concept(obs_code, obs_system) - observation.category = [make_concept("laboratory", "http://terminology.hl7.org/CodeSystem/observation-category")] + observation["code"] = make_concept(obs_code, obs_system) # lab result if obsfact.tval_char in external_mappings.SNOMED_LAB_RESULT: - lab_result = obsfact.tval_char + lab_result = external_mappings.SNOMED_LAB_RESULT[obsfact.tval_char] + lab_result_system = "http://snomed.info/sct" else: - lab_result = "Absent" - observation.valueCodeableConcept = make_concept( - external_mappings.SNOMED_LAB_RESULT[lab_result], "http://snomed.info/sct", display=lab_result - ) + lab_result = obsfact.tval_char + lab_result_system = "http://cumulus.smarthealthit.org/i2b2" + observation["valueCodeableConcept"] = make_concept(lab_result, lab_result_system, display=obsfact.tval_char) return observation -def to_fhir_condition(obsfact: ObservationFact) -> Condition: +def to_fhir_condition(obsfact: ObservationFact) -> dict: """ :param obsfact: i2b2 observation fact containing ICD9, ICD10, or SNOMED diagnosis :return: https://www.hl7.org/fhir/condition.html """ - condition = Condition() - condition.id = obsfact.instance_num - - condition.subject = ref_subject(obsfact.patient_num) - condition.encounter = ref_encounter(obsfact.encounter_num) - - condition.meta = Meta({"profile": ["http://hl7.org/fhir/us/core/StructureDefinition/us-core-condition"]}) - condition.recordedDate = parse_fhir_date(obsfact.start_date) - - condition.clinicalStatus = make_concept("active", "http://terminology.hl7.org/CodeSystem/condition-clinical") - condition.verificationStatus = make_concept( - "unconfirmed", "http://terminology.hl7.org/CodeSystem/condition-ver-status" - ) - - # Category - condition.category = [ - make_concept( - "encounter-diagnosis", - "http://terminology.hl7.org/CodeSystem/condition-category", - display="Encounter Diagnosis", - ) - ] + condition = { + "resourceType": "Condition", + "id": str(obsfact.instance_num), + "subject": fhir_common.ref_resource("Patient", obsfact.patient_num), + "encounter": fhir_common.ref_resource("Encounter", obsfact.encounter_num), + "meta": {"profile": ["http://hl7.org/fhir/us/core/StructureDefinition/us-core-condition"]}, + "category": [ + make_concept( + "encounter-diagnosis", + "http://terminology.hl7.org/CodeSystem/condition-category", + display="Encounter Diagnosis", + ) + ], + "recordedDate": chop_to_date(obsfact.start_date), + "clinicalStatus": make_concept("active", "http://terminology.hl7.org/CodeSystem/condition-clinical"), + "verificationStatus": make_concept("unconfirmed", "http://terminology.hl7.org/CodeSystem/condition-ver-status"), + } # Code i2b2_sys, i2b2_code = obsfact.concept_cd.split(":") if i2b2_sys in ["ICD10", "ICD-10"]: i2b2_sys = "http://hl7.org/fhir/sid/icd-10-cm" + elif i2b2_sys in ["ICD10PROC"]: + i2b2_sys = "http://hl7.org/fhir/sid/icd-10-pcs" elif i2b2_sys in ["ICD9", "ICD-9"]: i2b2_sys = "http://hl7.org/fhir/sid/icd-9-cm" + elif i2b2_sys in ["ICD9PROC"]: + i2b2_sys = "http://hl7.org/fhir/sid/icd-9-pcs" elif i2b2_sys in ["SNOMED", "SNOMED-CT", "SNOMEDCT", "SCT"]: i2b2_sys = "http://snomed.info/sct" else: @@ -224,7 +190,7 @@ def to_fhir_condition(obsfact: ObservationFact) -> Condition: i2b2_sys = "http://cumulus.smarthealthit.org/i2b2" i2b2_code = obsfact.concept_cd - condition.code = make_concept(i2b2_code, i2b2_sys) + condition["code"] = make_concept(i2b2_code, i2b2_sys) return condition @@ -236,39 +202,35 @@ def to_fhir_condition(obsfact: ObservationFact) -> Condition: ############################################################################### -def to_fhir_documentreference(obsfact: ObservationFact) -> DocumentReference: +def to_fhir_documentreference(obsfact: ObservationFact) -> dict: """ :param obsfact: i2b2 observation fact containing the I2b2 NOTE as OBSERVATION_BLOB :return: https://www.hl7.org/fhir/documentreference.html """ - docref = DocumentReference() - docref.indexed = FHIRDate() - - docref.id = obsfact.instance_num - docref.subject = ref_subject(obsfact.patient_num) - docref.context = DocumentReferenceContext() - docref.context.encounter = [ref_encounter(obsfact.encounter_num)] - docref.context.period = Period( - { - "start": parse_fhir_date_isostring(obsfact.start_date), - "end": parse_fhir_date_isostring(obsfact.end_date), - } - ) - - # It would be nice to get a real mapping for the "NOTE:" concept CD types to a real system. - # But for now, use this custom (and the URL isn't even valid) system to note these i2b2 concepts. - docref.type = make_concept(obsfact.concept_cd, "http://cumulus.smarthealthit.org/i2b2", obsfact.tval_char) - docref.status = "current" - blob = obsfact.observation_blob or "" - content = DocumentReferenceContent() - content.attachment = Attachment() - content.attachment.contentType = "text/plain" - content.attachment.data = base64.standard_b64encode(blob.encode("utf8")).decode("ascii") - docref.content = [content] - return docref + return { + "resourceType": "DocumentReference", + "id": str(obsfact.instance_num), + "subject": fhir_common.ref_resource("Patient", obsfact.patient_num), + "context": { + "encounter": [fhir_common.ref_resource("Encounter", obsfact.encounter_num)], + "period": {"start": chop_to_date(obsfact.start_date), "end": chop_to_date(obsfact.end_date)}, + }, + # It would be nice to get a real mapping for the "NOTE:" concept CD types to a real system. + # But for now, use this custom (and the URL isn't even valid) system to note these i2b2 concepts. + "type": make_concept(obsfact.concept_cd, "http://cumulus.smarthealthit.org/i2b2", obsfact.tval_char), + "status": "current", + "content": [ + { + "attachment": { + "contentType": "text/plain", + "data": base64.standard_b64encode(blob.encode("utf8")).decode("ascii"), + } + }, + ], + } ############################################################################### @@ -278,76 +240,17 @@ def to_fhir_documentreference(obsfact: ObservationFact) -> DocumentReference: ############################################################################### -def parse_zip_code(i2b2_zip_code) -> Optional[str]: - """ - :param i2b2_zip_code: - :return: Patient Address ZipCode (3-9 digits) - """ - if i2b2_zip_code and isinstance(i2b2_zip_code, str): - if 3 <= len(i2b2_zip_code) <= 9: - return i2b2_zip_code - - -def parse_gender(i2b2_sex_cd) -> Optional[str]: - """ - :param i2b2_sex_cd: - :return: FHIR AdministrativeGender code - """ - if i2b2_sex_cd and isinstance(i2b2_sex_cd, str): - return external_mappings.FHIR_GENDER.get(i2b2_sex_cd, "other") - - -def parse_fhir_duration(i2b2_length_of_stay) -> float: +def chop_to_date(yyyy_mm_dd: Optional[str]) -> Optional[str]: """ - :param i2b2_length_of_stay: usually an integer like "days" - :return: FHIR Duration float "time" - """ - if i2b2_length_of_stay: - if isinstance(i2b2_length_of_stay, str): - return float(i2b2_length_of_stay) - if isinstance(i2b2_length_of_stay, int): - return float(i2b2_length_of_stay) - if isinstance(i2b2_length_of_stay, float): - return i2b2_length_of_stay - + To be less sensitive to how i2b2 datetimes are formatted, chop to just the day/date part. -def parse_fhir_date(yyyy_mm_dd: str) -> Optional[FHIRDate]: - """ :param yyyy_mm_dd: YEAR Month Date - :return: FHIR Date with only the date part. + :return: only the date part. """ if yyyy_mm_dd and isinstance(yyyy_mm_dd, str): - yyyy_mm_dd = yyyy_mm_dd[:10] # ignore the time portion - return FHIRDate(yyyy_mm_dd) - - -def parse_fhir_date_isostring(yyyy_mm_dd: str) -> Optional[str]: - """ - :param yyyy_mm_dd: - :return: str version of the - """ - parsed = parse_fhir_date(yyyy_mm_dd) - return parsed.isostring if parsed else None - - -def ref_subject(subject_id: str) -> FHIRReference: - """ - Patient Reference the FHIR proper way - :param subject_id: ID for patient (isa REF can be UUID) - :return: FHIRReference as Patient/$id - """ - return FHIRReference(fhir_common.ref_resource("Patient", subject_id)) - - -def ref_encounter(encounter_id: str) -> FHIRReference: - """ - Encounter Reference the FHIR proper way - :param encounter_id: ID for encounter (isa REF can be UUID) - :return: FHIRReference as Encounter/$id - """ - return FHIRReference(fhir_common.ref_resource("Encounter", encounter_id)) + return yyyy_mm_dd[:10] # ignore the time portion -def make_concept(code: str, system: Optional[str], display: str = None) -> CodeableConcept: +def make_concept(code: str, system: Optional[str], display: str = None) -> dict: """Syntactic sugar to make a codeable concept""" - return CodeableConcept({"coding": [{"code": code, "system": system, "display": display}]}) + return {"coding": [{"code": code, "system": system, "display": display}]} diff --git a/tests/data/simple/batched-ndjson-output/observation/observation.001.ndjson b/tests/data/simple/batched-ndjson-output/observation/observation.001.ndjson index 68b3bfd4..bcd0d032 100644 --- a/tests/data/simple/batched-ndjson-output/observation/observation.001.ndjson +++ b/tests/data/simple/batched-ndjson-output/observation/observation.001.ndjson @@ -1 +1 @@ -{"id":"f29736c29af5b962b3947fd40bed6b8c3e97c642b72aaa08e082fec05148e7dd","category":[{"coding":[{"code":"laboratory","system":"http:\/\/terminology.hl7.org\/CodeSystem\/observation-category"}]}],"code":{"coding":[{"code":"94500-6","system":"http:\/\/loinc.org"}]},"effectiveDateTime":"2020-03-20","encounter":{"reference":"Encounter\/af1e6186-3f9a-1fa9-3c73-cfa56c84a056"},"status":"unknown","subject":{"reference":"Patient\/1de9ea66-70d3-da1f-c735-df5ef7697fb9"},"valueCodeableConcept":{"coding":[{"code":"272519000","display":"Absent","system":"http:\/\/snomed.info\/sct"}]},"resourceType":"Observation"} +{"id":"f29736c29af5b962b3947fd40bed6b8c3e97c642b72aaa08e082fec05148e7dd","category":[{"coding":[{"code":"laboratory","system":"http:\/\/terminology.hl7.org\/CodeSystem\/observation-category"}]}],"code":{"coding":[{"code":"LAB:1","system":"http:\/\/cumulus.smarthealthit.org\/i2b2"}]},"effectiveDateTime":"2020-03-20","encounter":{"reference":"Encounter\/af1e6186-3f9a-1fa9-3c73-cfa56c84a056"},"status":"unknown","subject":{"reference":"Patient\/1de9ea66-70d3-da1f-c735-df5ef7697fb9"},"valueCodeableConcept":{"coding":[{"code":"See Image","display":"See Image","system":"http:\/\/cumulus.smarthealthit.org\/i2b2"}]},"resourceType":"Observation"} diff --git a/tests/data/simple/i2b2-input/observation_fact_lab_views.csv b/tests/data/simple/i2b2-input/observation_fact_lab_views.csv index f8d99c8d..d1690b5e 100644 --- a/tests/data/simple/i2b2-input/observation_fact_lab_views.csv +++ b/tests/data/simple/i2b2-input/observation_fact_lab_views.csv @@ -1,3 +1,3 @@ "ENCOUNTER_NUM","PATIENT_NUM","CONCEPT_CD","PROVIDER_ID","START_DATE","MODIFIER_CD","INSTANCE_NUM","VALTYPE_CD","TVAL_CHAR","NVAL_NUM","VALUEFLAG_CD","QUANTITY_NUM","UNITS_CD","END_DATE","LOCATION_CD","OBSERVATION_BLOB","CONFIDENCE_NUM","UPDATE_DATE","DOWNLOAD_DATE","IMPORT_DATE","SOURCESYSTEM_CD","UPLOAD_ID","TEXT_SEARCH_INDEX" -22,323456,LAB:1043473617,"52",2020-03-19 01:50:00,@,42,T,See Image,,@,,NOT DEFINED IN SOURCE,2020-03-19 01:50:00,,"",,,,2021-05-29 00:00:00,LAB,, -25,323456,LAB:1043473617,"52",2020-03-20 01:50:00,@,43,T,See Image,,@,,NOT DEFINED IN SOURCE,2020-03-20 01:50:00,,"",,,,2021-05-30 00:00:00,LAB,, +22,323456,LAB:1043473617,"52",2020-03-19 01:50:00,@,42,T,Absent,,@,,NOT DEFINED IN SOURCE,2020-03-19 01:50:00,,"",,,,2021-05-29 00:00:00,LAB,, +25,323456,LAB:1,"52",2020-03-20 01:50:00,@,43,T,See Image,,@,,NOT DEFINED IN SOURCE,2020-03-20 01:50:00,,"",,,,2021-05-30 00:00:00,LAB,, diff --git a/tests/data/simple/ndjson-input/Observation.ndjson b/tests/data/simple/ndjson-input/Observation.ndjson index 90d6ef16..7d59c17e 100644 --- a/tests/data/simple/ndjson-input/Observation.ndjson +++ b/tests/data/simple/ndjson-input/Observation.ndjson @@ -1,2 +1,2 @@ {"id":"42","category":[{"coding":[{"code":"laboratory","system":"http://terminology.hl7.org/CodeSystem/observation-category"}]}],"code":{"coding":[{"code":"94500-6","system":"http:\/\/loinc.org"}]},"effectiveDateTime":"2020-03-19","encounter":{"reference":"Encounter\/22"},"status":"unknown","subject":{"reference":"Patient\/323456"},"valueCodeableConcept":{"coding":[{"code":"272519000","display":"Absent","system":"http:\/\/snomed.info\/sct"}]},"resourceType":"Observation"} -{"id":"43","category":[{"coding":[{"code":"laboratory","system":"http://terminology.hl7.org/CodeSystem/observation-category"}]}],"code":{"coding":[{"code":"94500-6","system":"http:\/\/loinc.org"}]},"effectiveDateTime":"2020-03-20","encounter":{"reference":"Encounter\/25"},"status":"unknown","subject":{"reference":"Patient\/323456"},"valueCodeableConcept":{"coding":[{"code":"272519000","display":"Absent","system":"http:\/\/snomed.info\/sct"}]},"resourceType":"Observation"} +{"id":"43","category":[{"coding":[{"code":"laboratory","system":"http://terminology.hl7.org/CodeSystem/observation-category"}]}],"code":{"coding":[{"code":"LAB:1","system":"http:\/\/cumulus.smarthealthit.org\/i2b2"}]},"effectiveDateTime":"2020-03-20","encounter":{"reference":"Encounter\/25"},"status":"unknown","subject":{"reference":"Patient\/323456"},"valueCodeableConcept":{"coding":[{"code":"See Image","display":"See Image","system":"http:\/\/cumulus.smarthealthit.org\/i2b2"}]},"resourceType":"Observation"} diff --git a/tests/data/simple/ndjson-output/observation/observation.000.ndjson b/tests/data/simple/ndjson-output/observation/observation.000.ndjson index 978157be..20e28ba4 100644 --- a/tests/data/simple/ndjson-output/observation/observation.000.ndjson +++ b/tests/data/simple/ndjson-output/observation/observation.000.ndjson @@ -1,2 +1,2 @@ {"id":"76da69dede003b4ceff5dc4921f838f3f8e583ef1e999cedc4bbe30c4d6d0940","category":[{"coding":[{"code":"laboratory","system":"http:\/\/terminology.hl7.org\/CodeSystem\/observation-category"}]}],"code":{"coding":[{"code":"94500-6","system":"http:\/\/loinc.org"}]},"effectiveDateTime":"2020-03-19","encounter":{"reference":"Encounter\/175e9941-2607-ad5f-76ab-14759da618fd"},"status":"unknown","subject":{"reference":"Patient\/1de9ea66-70d3-da1f-c735-df5ef7697fb9"},"valueCodeableConcept":{"coding":[{"code":"272519000","display":"Absent","system":"http:\/\/snomed.info\/sct"}]},"resourceType":"Observation"} -{"id":"f29736c29af5b962b3947fd40bed6b8c3e97c642b72aaa08e082fec05148e7dd","category":[{"coding":[{"code":"laboratory","system":"http:\/\/terminology.hl7.org\/CodeSystem\/observation-category"}]}],"code":{"coding":[{"code":"94500-6","system":"http:\/\/loinc.org"}]},"effectiveDateTime":"2020-03-20","encounter":{"reference":"Encounter\/af1e6186-3f9a-1fa9-3c73-cfa56c84a056"},"status":"unknown","subject":{"reference":"Patient\/1de9ea66-70d3-da1f-c735-df5ef7697fb9"},"valueCodeableConcept":{"coding":[{"code":"272519000","display":"Absent","system":"http:\/\/snomed.info\/sct"}]},"resourceType":"Observation"} +{"id":"f29736c29af5b962b3947fd40bed6b8c3e97c642b72aaa08e082fec05148e7dd","category":[{"coding":[{"code":"laboratory","system":"http:\/\/terminology.hl7.org\/CodeSystem\/observation-category"}]}],"code":{"coding":[{"code":"LAB:1","system":"http:\/\/cumulus.smarthealthit.org\/i2b2"}]},"effectiveDateTime":"2020-03-20","encounter":{"reference":"Encounter\/af1e6186-3f9a-1fa9-3c73-cfa56c84a056"},"status":"unknown","subject":{"reference":"Patient\/1de9ea66-70d3-da1f-c735-df5ef7697fb9"},"valueCodeableConcept":{"coding":[{"code":"See Image","display":"See Image","system":"http:\/\/cumulus.smarthealthit.org\/i2b2"}]},"resourceType":"Observation"} diff --git a/tests/i2b2_mock_data.py b/tests/i2b2_mock_data.py index ddb77bb2..3bb01be5 100644 --- a/tests/i2b2_mock_data.py +++ b/tests/i2b2_mock_data.py @@ -16,7 +16,7 @@ def patient_dim() -> transform.PatientDimension: ) -def patient() -> transform.Patient: +def patient() -> dict: return transform.to_fhir_patient(patient_dim()) @@ -33,7 +33,7 @@ def encounter_dim() -> transform.VisitDimension: ) -def encounter() -> transform.Encounter: +def encounter() -> dict: return transform.to_fhir_encounter(encounter_dim()) @@ -49,7 +49,7 @@ def condition_dim() -> transform.ObservationFact: ) -def condition() -> transform.Condition: +def condition() -> dict: return transform.to_fhir_condition(condition_dim()) @@ -67,7 +67,7 @@ def documentreference_dim() -> transform.ObservationFact: ) -def documentreference() -> transform.DocumentReference: +def documentreference() -> dict: return transform.to_fhir_documentreference(documentreference_dim()) @@ -85,5 +85,5 @@ def observation_dim() -> transform.ObservationFact: ) -def observation() -> transform.Observation: +def observation() -> dict: return transform.to_fhir_observation_lab(observation_dim()) diff --git a/tests/test_i2b2_oracle_extract.py b/tests/test_i2b2_oracle_extract.py index 3825acd5..ae623411 100644 --- a/tests/test_i2b2_oracle_extract.py +++ b/tests/test_i2b2_oracle_extract.py @@ -102,12 +102,6 @@ def test_loader(self, mock_extract): set(os.listdir(tmpdir.name)), ) - self.assertEqual( - i2b2_mock_data.condition().as_json(), common.read_json(os.path.join(tmpdir.name, "Condition.ndjson")) - ) - self.assertEqual( - i2b2_mock_data.encounter().as_json(), common.read_json(os.path.join(tmpdir.name, "Encounter.ndjson")) - ) - self.assertEqual( - i2b2_mock_data.patient().as_json(), common.read_json(os.path.join(tmpdir.name, "Patient.ndjson")) - ) + self.assertEqual(i2b2_mock_data.condition(), common.read_json(os.path.join(tmpdir.name, "Condition.ndjson"))) + self.assertEqual(i2b2_mock_data.encounter(), common.read_json(os.path.join(tmpdir.name, "Encounter.ndjson"))) + self.assertEqual(i2b2_mock_data.patient(), common.read_json(os.path.join(tmpdir.name, "Patient.ndjson"))) diff --git a/tests/test_i2b2_transform.py b/tests/test_i2b2_transform.py index 4bd63dc5..f9f17298 100644 --- a/tests/test_i2b2_transform.py +++ b/tests/test_i2b2_transform.py @@ -3,7 +3,6 @@ import unittest import ddt -from fhirclient.models.fhirdate import FHIRDate from cumulus.loaders.i2b2 import transform from tests import i2b2_mock_data @@ -13,19 +12,15 @@ class TestI2b2Transform(unittest.TestCase): """Test case for converting from i2b2 to FHIR""" - # Pylint doesn't like subscripting some lists in our created objects, not sure why yet. - # pylint: disable=unsubscriptable-object - def test_to_fhir_patient(self): subject = i2b2_mock_data.patient() + # print(json.dumps(subject, indent=4)) - # print(json.dumps(pat_fhir.as_json(), indent=4)) - - self.assertEqual(str(12345), subject.id) - self.assertEqual("2005-06-07", subject.birthDate.isostring) - self.assertEqual("female", subject.gender) + self.assertEqual(str(12345), subject["id"]) + self.assertEqual("2005-06-07", subject["birthDate"]) + self.assertEqual("female", subject["gender"]) # pylint: disable-next=unsubscriptable-object - self.assertEqual("02115", subject.address[0].postalCode) + self.assertEqual("02115", subject["address"][0]["postalCode"]) @ddt.data( ("Black or African American", "race", "urn:oid:2.16.840.1.113883.6.238", "2054-5"), @@ -52,77 +47,49 @@ def test_patient_race_vs_ethnicity(self, race_cd, url, system, code): }, ], }, - patient.extension[0].as_json(), + patient["extension"][0], ) def test_to_fhir_encounter(self): encounter = i2b2_mock_data.encounter() - # print(json.dumps(encounter.as_json(), indent=4)) + # print(json.dumps(encounter, indent=4)) - self.assertEqual("67890", encounter.id) - self.assertEqual("Patient/12345", encounter.subject.reference) - self.assertEqual("2016-01-01", encounter.period.start.isostring) - self.assertEqual("2016-01-04", encounter.period.end.isostring) - self.assertEqual(3, encounter.length.value) + self.assertEqual("67890", encounter["id"]) + self.assertEqual("Patient/12345", encounter["subject"]["reference"]) + self.assertEqual("2016-01-01", encounter["period"]["start"]) + self.assertEqual("2016-01-04", encounter["period"]["end"]) + self.assertEqual(3, encounter["length"]["value"]) def test_to_fhir_condition(self): condition = i2b2_mock_data.condition() + # print(json.dumps(condition, indent=4)) - # print(json.dumps(condition.as_json(), indent=4)) - self.assertEqual("Patient/12345", condition.subject.reference) - self.assertEqual("Encounter/67890", condition.encounter.reference) - self.assertEqual(str("U07.1"), condition.code.coding[0].code) - self.assertEqual(str("http://hl7.org/fhir/sid/icd-10-cm"), condition.code.coding[0].system) + self.assertEqual("Patient/12345", condition["subject"]["reference"]) + self.assertEqual("Encounter/67890", condition["encounter"]["reference"]) + self.assertEqual("U07.1", condition["code"]["coding"][0]["code"]) + self.assertEqual("http://hl7.org/fhir/sid/icd-10-cm", condition["code"]["coding"][0]["system"]) def test_to_fhir_documentreference(self): docref = i2b2_mock_data.documentreference() + # print(json.dumps(docref, indent=4)) - # print(json.dumps(docref.as_json(), indent=4)) - - self.assertEqual("Patient/12345", docref.subject.reference) - self.assertEqual(1, len(docref.context.encounter)) - self.assertEqual("Encounter/67890", docref.context.encounter[0].reference) - self.assertEqual("NOTE:149798455", docref.type.coding[0].code) - self.assertEqual("Emergency note", docref.type.coding[0].display) + self.assertEqual("Patient/12345", docref["subject"]["reference"]) + self.assertEqual(1, len(docref["context"]["encounter"])) + self.assertEqual("Encounter/67890", docref["context"]["encounter"][0]["reference"]) + self.assertEqual("NOTE:149798455", docref["type"]["coding"][0]["code"]) + self.assertEqual("Emergency note", docref["type"]["coding"][0]["display"]) def test_to_fhir_observation_lab(self): lab_fhir = i2b2_mock_data.observation() + # print(json.dumps(lab_fhir, indent=4)) - # print(json.dumps(lab_i2b2.__dict__, indent=4)) - # print(json.dumps(lab_fhir.as_json(), indent=4)) - - self.assertEqual("Patient/12345", lab_fhir.subject.reference) - self.assertEqual("Encounter/67890", lab_fhir.encounter.reference) - - self.assertEqual("94500-6", lab_fhir.code.coding[0].code) - self.assertEqual("http://loinc.org", lab_fhir.code.coding[0].system) - - self.assertEqual("260385009", lab_fhir.valueCodeableConcept.coding[0].code) - self.assertEqual("Negative", lab_fhir.valueCodeableConcept.coding[0].display) - - self.assertEqual(FHIRDate("2021-01-02").date, lab_fhir.effectiveDateTime.date) - - def test_parse_fhir_date(self): - - timestamp = "2020-01-02 12:00:00.000" - timestamp = timestamp[:10] - - self.assertEqual("2020-01-02", FHIRDate(timestamp).isostring) - - timestamp = "2020-01-02 12:00:00.000" - - self.assertEqual("2020-01-02", transform.parse_fhir_date(timestamp).isostring) - - timezone = "2020-01-02T16:00:00+00:00" - - self.assertEqual("2020-01-02", transform.parse_fhir_date(timezone).isostring) - - datepart = "2020-01-02" + self.assertEqual("Patient/12345", lab_fhir["subject"]["reference"]) + self.assertEqual("Encounter/67890", lab_fhir["encounter"]["reference"]) - self.assertEqual("2020-01-02", transform.parse_fhir_date(datepart).isostring) + self.assertEqual("94500-6", lab_fhir["code"]["coding"][0]["code"]) + self.assertEqual("http://loinc.org", lab_fhir["code"]["coding"][0]["system"]) - def test_ref_subject(self): - self.assertEqual({"reference": "Patient/123"}, transform.ref_subject("123").as_json()) + self.assertEqual("260385009", lab_fhir["valueCodeableConcept"]["coding"][0]["code"]) + self.assertEqual("Negative", lab_fhir["valueCodeableConcept"]["coding"][0]["display"]) - def test_ref_encounter(self): - self.assertEqual({"reference": "Encounter/123"}, transform.ref_encounter("123").as_json()) + self.assertEqual("2021-01-02", lab_fhir["effectiveDateTime"]) diff --git a/tests/test_scrubber.py b/tests/test_scrubber.py index a6842eb5..14099e85 100644 --- a/tests/test_scrubber.py +++ b/tests/test_scrubber.py @@ -17,7 +17,7 @@ class TestScrubber(unittest.TestCase): def test_patient(self): """Verify a basic patient (saved ids)""" - patient = i2b2_mock_data.patient().as_json() + patient = i2b2_mock_data.patient() self.assertEqual("12345", patient["id"]) scrubber = Scrubber() @@ -26,7 +26,7 @@ def test_patient(self): def test_encounter(self): """Verify a basic encounter (saved ids)""" - encounter = i2b2_mock_data.encounter().as_json() + encounter = i2b2_mock_data.encounter() self.assertEqual("Patient/12345", encounter["subject"]["reference"]) self.assertEqual("67890", encounter["id"]) @@ -37,7 +37,7 @@ def test_encounter(self): def test_condition(self): """Verify a basic condition (hashed ids)""" - condition = i2b2_mock_data.condition().as_json() + condition = i2b2_mock_data.condition() self.assertEqual("4567", condition["id"]) self.assertEqual("Patient/12345", condition["subject"]["reference"]) self.assertEqual("Encounter/67890", condition["encounter"]["reference"]) @@ -52,7 +52,7 @@ def test_condition(self): def test_documentreference(self): """Test DocumentReference, which is interesting because of its list of encounters and attachments""" - docref = i2b2_mock_data.documentreference().as_json() + docref = i2b2_mock_data.documentreference() self.assertEqual("345", docref["id"]) self.assertEqual("Patient/12345", docref["subject"]["reference"]) self.assertEqual(1, len(docref["context"]["encounter"])) @@ -72,7 +72,7 @@ def test_documentreference(self): def test_unknown_modifier_extension(self): """Confirm we skip resources with unknown modifier extensions""" - patient = i2b2_mock_data.patient().as_json() + patient = i2b2_mock_data.patient() scrubber = Scrubber() patient["modifierExtension"] = [] @@ -104,7 +104,7 @@ def test_load_and_save(self): # Confirm we loaded that encounter correctly scrubber = Scrubber(tmpdir) - encounter = i2b2_mock_data.encounter().as_json() # patient is 12345 + encounter = i2b2_mock_data.encounter() # patient is 12345 encounter["id"] = "1" self.assertTrue(scrubber.scrub_resource(encounter)) self.assertEqual(encounter["id"], db.encounter("1")) @@ -131,7 +131,7 @@ def test_load_and_save(self): def test_meta_security_cleared(self): """Verify that we drop the Meta.security field""" scrubber = Scrubber() - condition = i2b2_mock_data.condition().as_json() + condition = i2b2_mock_data.condition() # With another property condition["meta"] = {"security": [{"code": "REDACTED"}], "versionId": "a"} diff --git a/tests/test_tasks.py b/tests/test_tasks.py index 4bba3fe4..aabdfdb1 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -6,8 +6,6 @@ import unittest from unittest import mock -from fhirclient.models.extension import Extension - from cumulus import common, config, deid, errors, store, tasks from tests.ctakesmock import CtakesMixin @@ -134,12 +132,12 @@ def test_unknown_modifier_extensions_skipped_for_patients(self): def test_unknown_modifier_extensions_skipped_for_nlp_symptoms(self): """Verify we ignore unknown modifier extensions during a custom read task (nlp symptoms)""" docref0 = i2b2_mock_data.documentreference() - docref0.subject.reference = "Patient/1234" - self.make_json("DocumentReference.0", "0", **docref0.as_json()) + docref0["subject"]["reference"] = "Patient/1234" + self.make_json("DocumentReference.0", "0", **docref0) docref1 = i2b2_mock_data.documentreference() - docref1.subject.reference = "Patient/5678" - docref1.modifierExtension = [Extension({"url": "unrecognized"})] - self.make_json("DocumentReference.1", "1", **docref1.as_json()) + docref1["subject"]["reference"] = "Patient/5678" + docref1["modifierExtension"] = [{"url": "unrecognized"}] + self.make_json("DocumentReference.1", "1", **docref1) tasks.CovidSymptomNlpResultsTask(self.job_config, self.scrubber).run() @@ -152,11 +150,11 @@ def test_unknown_modifier_extensions_skipped_for_nlp_symptoms(self): def test_non_ed_visit_is_skipped_for_covid_symptoms(self): """Verify we ignore non ED visits for the covid symptoms NLP""" docref0 = i2b2_mock_data.documentreference() - docref0.type.coding[0].code = "NOTE:nope" # pylint: disable=unsubscriptable-object - self.make_json("DocumentReference.0", "skipped", **docref0.as_json()) + docref0["type"]["coding"][0]["code"] = "NOTE:nope" # pylint: disable=unsubscriptable-object + self.make_json("DocumentReference.0", "skipped", **docref0) docref1 = i2b2_mock_data.documentreference() - docref1.type.coding[0].code = "NOTE:149798455" # pylint: disable=unsubscriptable-object - self.make_json("DocumentReference.1", "present", **docref1.as_json()) + docref1["type"]["coding"][0]["code"] = "NOTE:149798455" # pylint: disable=unsubscriptable-object + self.make_json("DocumentReference.1", "present", **docref1) tasks.CovidSymptomNlpResultsTask(self.job_config, self.scrubber).run()