diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index b22ac48..0452373 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -19,10 +19,21 @@ jobs: uses: actions/setup-python@v1 with: python-version: 2.7 + - name: Add dummy genologicsrc + run: | + cat << EOF > ~/.genologicsrc + [genologics] + BASEURI=https://fancy.server + USERNAME=username + PASSWORD=pass + EOF + cat ~/.genologicsrc - name: Install dependencies run: | ./clarity-ext-scripts/setup.sh - name: Test with pytest run: | pip install pytest - pytest ./clarity-ext-scripts/tests + pytest ./clarity-ext-scripts/tests/unit + pytest ./sminet-client/tests/unit + diff --git a/README.md b/README.md index 12912c9..c506f3e 100644 --- a/README.md +++ b/README.md @@ -68,5 +68,9 @@ The [clarity-ext](https://github.com/molmed/clarity-ext) package is designed to Extensions that use this framework are all in ./clarity-ext-scripts. Refer to the [README](./clarity-ext-scripts/README.md) for more information on this package. +## SmiNet + +There is a client for SmiNet integration in ./sminet_client/. Refer to the [README](./sminet_client/README.md) for more information on the package. + ## More information More information about the LIMS system including details on the Miniconda installation and starting the server may be found [here](https://github.com/ctmrbio/wiki/wiki/CTMR-LIMS-(PROD-and-STAGE)). diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/create_samples/assign_unregistered_to_anonymous.py b/clarity-ext-scripts/clarity_ext_scripts/covid/create_samples/assign_unregistered_to_anonymous.py index f811719..9ae8d40 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/create_samples/assign_unregistered_to_anonymous.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/create_samples/assign_unregistered_to_anonymous.py @@ -1,9 +1,9 @@ from uuid import uuid4 from clarity_ext.extensions import GeneralExtension from clarity_ext_scripts.covid.create_samples.common import ValidatedSampleListFile -from clarity_ext_scripts.covid.utils import KNMClient +from clarity_ext_scripts.covid.services.knm_service import KNMClientFromExtension from clarity_ext_scripts.covid.partner_api_client import ( - ORG_URI_BY_NAME, TESTING_ORG, CouldNotCreateServiceRequest, ServiceRequestAlreadyExists) + ORG_URI_BY_NAME, TESTING_ORG, CouldNotCreateServiceRequest, ServiceRequestAlreadyExists) class Extension(GeneralExtension): @@ -11,7 +11,7 @@ class Extension(GeneralExtension): Goes through the 'Validated sample list' ensuring that each sample that has the unregistered status gets an anonymous service request id. - Uploads a new file to the validated sample list file handle. The name of it will be on the form + Uploads a new file to the validated sample list file handle. The name of it will be on the form 'validated_sample_list_no_unregistered_.csv' Note that the user is not required to run this extension. They can create samples directly @@ -20,7 +20,7 @@ class Extension(GeneralExtension): """ def execute(self): - client = KNMClient(self) + client = KNMClientFromExtension(self) validated_sample_list = ValidatedSampleListFile.create_from_context( self.context) no_unregistered = ValidatedSampleListFile(validated_sample_list.csv) diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/discard_samples.py b/clarity-ext-scripts/clarity_ext_scripts/covid/discard_samples.py index 36c21a0..5b1a123 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/discard_samples.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/discard_samples.py @@ -1,12 +1,9 @@ import logging from datetime import datetime -from clarity_ext.extensions import GeneralExtension from clarity_ext_scripts.covid.partner_api_client import ( - PartnerAPIV7Client, TESTING_ORG, ORG_URI_BY_NAME, COVID_RESPONSE_FAILED, - PartnerClientAPIException) -from clarity_ext_scripts.covid.rtpcr_analysis_service import FAILED_STATES -from clarity_ext_scripts.covid.controls import Controls -from clarity_ext_scripts.covid.utils import CtmrCovidSubstanceInfo, KNMClient + TESTING_ORG, ORG_URI_BY_NAME, COVID_RESPONSE_FAILED, + PartnerClientAPIException) +from clarity_ext_scripts.covid.utils import CtmrCovidSubstanceInfo, KNMClientFromExtension from clarity_ext_scripts.covid.import_samples import BaseCreateSamplesExtension logger = logging.getLogger(__name__) @@ -61,18 +58,20 @@ def report(self, analyte): self.context.commit() def execute(self): - self.client = KNMClient(self) + self.client = KNMClientFromExtension(self) for plate in self.context.input_containers: for well in plate.occupied: already_uploaded = False try: already_uploaded = well.artifact.udf_knm_result_uploaded == UDF_TRUE + except AttributeError: pass if already_uploaded: logger.info("Analyte {} has already been uploaded".format( well.artifact.name)) + continue self.report(well.artifact) diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/import_samples.py b/clarity-ext-scripts/clarity_ext_scripts/covid/import_samples.py index 00b73fc..771b493 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/import_samples.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/import_samples.py @@ -1,6 +1,6 @@ from clarity_ext.domain import Container, Sample from clarity_ext_scripts.covid.controls import controls_barcode_generator, Controls -from clarity_ext_scripts.covid.utils import KNMClient +from clarity_ext_scripts.covid.services.knm_service import KNMClientFromExtension from clarity_ext_scripts.covid.create_samples.common import BaseCreateSamplesExtension @@ -118,7 +118,7 @@ def execute(self): """ Creates samples from a validated import file """ - self.client = KNMClient(self) + self.client = KNMClientFromExtension(self) # This is for debug reasons only. Set this to True to create samples even if they have # been created before. This will overwrite the field udf_created_containers. diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/partner_api_client.py b/clarity-ext-scripts/clarity_ext_scripts/covid/partner_api_client.py index 0af08a3..fc75ab5 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/partner_api_client.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/partner_api_client.py @@ -1,4 +1,3 @@ - import base64 from datetime import datetime from luhn import verify as mod10verify @@ -400,6 +399,25 @@ def post_diagnosis_report(self, service_request_id, diagnosis_result, analysis_r log.error(e.message) raise e + def get_by_reference(self, ref): + """ + Get's a resource by reference, such as Organization/123 or Patient/345 + + :ref: The reference, e.g. Patient/123 + """ + url = "{}/{}".format(self._base_url, ref) + headers = self._generate_headers() + response = self._session.get(url=url, headers=headers) + if response.status_code != 200: + print(response.text) + raise PartnerClientAPIException( + "Couldn't get resource '{}', status code: {}".format( + url, response.status_code)) + return response.json() + + def get_org_uri_by_name(self, name): + return ORG_URI_BY_NAME[name] + def _create_payload(self, service_request_id, diagnosis_result, analysis_results): # TODO Need to think about if this needs refactoring later, to more easily support multiple # analysis types. This is going to be rather unwieldy to add more as it is implemented diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/report_results.py b/clarity-ext-scripts/clarity_ext_scripts/covid/report_results.py index 9eaadf1..5614204 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/report_results.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/report_results.py @@ -6,7 +6,7 @@ PartnerClientAPIException) from clarity_ext_scripts.covid.rtpcr_analysis_service import FAILED_STATES from clarity_ext_scripts.covid.controls import Controls -from clarity_ext_scripts.covid.utils import KNMClient +from clarity_ext_scripts.covid.services.knm_service import KNMClientFromExtension logger = logging.getLogger(__name__) @@ -70,7 +70,7 @@ def is_control(self, sample): return False def execute(self): - self.client = KNMClient(self) + self.client = KNMClientFromExtension(self) for plate in self.context.input_containers: for well in plate.occupied: already_uploaded = False diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/report_to_sminet.py b/clarity-ext-scripts/clarity_ext_scripts/covid/report_to_sminet.py new file mode 100644 index 0000000..7d92169 --- /dev/null +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/report_to_sminet.py @@ -0,0 +1,131 @@ +from datetime import datetime +import logging +from sminet_client import SampleMaterial, SmiNetError +from clarity_ext.extensions import GeneralExtension +from clarity_ext_scripts.covid.utils import CtmrCovidSubstanceInfo +from clarity_ext_scripts.covid.services.sminet_service import SmiNetService +from clarity_ext_scripts.covid.services.knm_service import KNMSampleAccessor +from clarity_ext_scripts.covid.services.knm_sminet_service import ( + KNMSmiNetIntegrationService, IntegrationError, UnregisteredPatient) +from clarity_ext_scripts.covid.partner_api_client import ( + COVID_RESPONSE_POSITIVE, PartnerClientAPIException) + + +UDF_TRUE = "Yes" +logger = logging.getLogger(__name__) + + +def should_report(substance): + """ + A substance is ignored from SmiNet reporting if it's a control, has been run before or doesn't + have a positive result. It is also ignored if the research engineer has set the UDF + "SmiNet status" to "ignore" + + Samples that should be reported must be in the SmiNet state `retry` or `error` or None + """ + if substance.is_control: + return False + + if substance.submitted_sample.udf_rtpcr_covid19_result_latest != COVID_RESPONSE_POSITIVE: + return False + + if substance.sminet_status in [SmiNetService.STATUS_IGNORE, SmiNetService.STATUS_SUCCESS]: + return False + + if substance.sminet_status not in [ + None, + SmiNetService.STATUS_RETRY, + SmiNetService.STATUS_ERROR]: + raise AssertionError( + "Unexpected SmiNet status: {}".format(substance.sminet_status)) + + return True + + +class Extension(GeneralExtension): + """ + Reports results to SmiNet. + + For all analytes in the step: + * If sminet_status is not set: + * Checks if it should be imported. + * If it shouldn't, updates it to "ignore" + * If it should, tries to send the data + * If sminet_status is "error", retries it + * If sminet_status is "ignore", ignores it + + + Test data required: + * Positive => success + * Negative => ignore + * Failed => ignore + * Anonymous => ignore + """ + + def report(self, substance): + """ + Reports this substance to SmiNet and updates the status in the LIMS. + """ + + org_referral_code = substance.submitted_sample.name.split("_")[0] + date_arrival = substance.submitted_sample.api_resource.date_received + date_arrival = datetime.strptime(date_arrival, "%Y-%m-%d") + + sample = KNMSampleAccessor(substance.submitted_sample.udf_knm_org_uri, + org_referral_code, + date_arrival, + SampleMaterial.SVALG) + + integration = KNMSmiNetIntegrationService(self.config) + lab_result = SmiNetService.create_scov2_positive_lab_result() + + error_msg = "" + + try: + integration.export_to_sminet(sample, + doctor_name="Lars Engstrand", + lab_result=lab_result, + sample_free_text="Anamnes: Personalprov") + status = SmiNetService.STATUS_SUCCESS + except UnregisteredPatient: + status = SmiNetService.STATUS_IGNORE + except PartnerClientAPIException as knm_error: + error_msg = knm_error.message + self.usage_error_defer("Error while fetching data from KNM", + substance.submitted_sample.name) + status = SmiNetService.STATUS_ERROR + except SmiNetError as sminet_error: + error_msg = sminet_error.message + self.usage_error_defer("Error while uploading sample to SmiNet", + substance.submitted_sample.name) + status = SmiNetService.STATUS_ERROR + except IntegrationError as int_error: + self.usage_error_defer("Error while uploading sample to SmiNet", + substance.submitted_sample.name) + error_msg = int_error.message + status = SmiNetService.STATUS_ERROR + + timestamp = datetime.now().strftime("%y%m%dT%H%M%S") + + substance.submitted_sample.udf_map.force("SmiNet status", status) + substance.substance.udf_map.force("SmiNet status", status) + + substance.submitted_sample.udf_map.force( + "SmiNet uploaded date", timestamp) + substance.submitted_sample.udf_map.force("SmiNet artifact source", + substance.substance.api_resource.uri) + substance.submitted_sample.udf_map.force( + "SmiNet last error", error_msg) + self.context.update(substance.submitted_sample) + self.context.update(substance.substance) + self.context.commit() + + def execute(self): + for plate in self.context.input_containers: + for well in plate.occupied: + substance = CtmrCovidSubstanceInfo(well.artifact) + if should_report(substance): + self.report(substance) + + def integration_tests(self): + yield "24-48808" diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/services/__init__.py b/clarity-ext-scripts/clarity_ext_scripts/covid/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/services/knm_service.py b/clarity-ext-scripts/clarity_ext_scripts/covid/services/knm_service.py new file mode 100644 index 0000000..7ce2473 --- /dev/null +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/services/knm_service.py @@ -0,0 +1,92 @@ +from clarity_ext_scripts.covid.partner_api_client import PartnerAPIV7Client +from clarity_ext.utils import lazyprop + + +class NoSupportedCountyCodeFound(Exception): + pass + + +class KNMService(object): + NoSupportedCountyCodeFound = NoSupportedCountyCodeFound + + def __init__(self, config): + self.config = KNMConfig(config) + self.client = PartnerAPIV7Client(**self.config) + + +def KNMConfig(config): + """ + Creates config required for KNM from the clarity-ext config (which has more than that) + """ + return { + key: config[key] + for key in [ + "test_partner_base_url", + "test_partner_code_system_base_url", + "test_partner_user", + "test_partner_password" + ] + } + + +def KNMClientFromExtension(extension): + # A factory for a KnmClient from an extension + config = KNMConfig(extension.config) + return PartnerAPIV7Client(**config) + + +class KNMSampleAccessor(object): + """ + Describes a sample/analyte in the LIMS that has originally come via the KNM workflow + """ + + def __init__(self, org_uri, org_referral_code, date_arrival, material): + """ + :org_uri: The organization URI as defined by KNM + :org_referral_code: The referral code within the organization + :date_arrival: The date the sample was added to the LIMS + :material: The material, one of the constants in SampleMaterialType + """ + self.org_uri = org_uri + self.org_referral_code = org_referral_code + self.date_arrival = date_arrival + self.material = material + + @classmethod + def create_from_lims_sample(cls, sample): + raise NotImplementedError() + + +class ServiceRequestProvider(object): + """ + Represents a service request. Has methods to retrieve all data we require on it. Gives + a higher level abstraction of the api for readability. + """ + + def __init__(self, client, org_uri, org_referral_code): + self.client = client + self.org_uri = org_uri + self.org_referral_code = org_referral_code + + @lazyprop + def service_request(self): + # The service_request json response + return self.client.search_for_service_request(self.org_uri, self.org_referral_code) + + @lazyprop + def patient(self): + # The patient data corresponding with the service request + return self.client.get_by_reference(self.patient_ref) + + @lazyprop + def organization(self): + managing_organization = self.patient["managingOrganization"] + return self.client.get_by_reference(managing_organization["reference"]) + + @property + def patient_ref(self): + # A reference identifying the patient + return self.service_request["resource"]["subject"]["reference"] + + def __str__(self): + return "{}|{}".format(self.org_uri, self.org_referral_code) diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/services/knm_sminet_service.py b/clarity-ext-scripts/clarity_ext_scripts/covid/services/knm_sminet_service.py new file mode 100644 index 0000000..d1bdf52 --- /dev/null +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/services/knm_sminet_service.py @@ -0,0 +1,142 @@ +import dateutil.parser +from datetime import datetime +from sminet_client import (SampleInfo, ReferringClinic, Patient, Doctor, + SmiNetLabExport, StatusType, Notification) +from sminet_client import Gender as SmiNetGender +from clarity_ext_scripts.covid.services.knm_service import ServiceRequestProvider, KNMService +from clarity_ext_scripts.covid.services.sminet_service import SmiNetService + + +class KNMSmiNetIntegrationService(object): + + MAP_GENDER_FROM_KNM_TO_SMINET = { + "male": SmiNetGender.MALE, + "female": SmiNetGender.FEMALE, + "other": SmiNetGender.UNKNOWN, + "unknown": SmiNetGender.UNKNOWN, + None: SmiNetGender.UNKNOWN, + } + + def __init__(self, config, knm_service=None, sminet_service=None): + self.config = config + self.knm_service = knm_service or KNMService(config) + self.sminet_service = sminet_service or SmiNetService(config) + + def get_county_from_organization(self, organization): + """ + Given the raw json from the KNM service, returns a county understood by SmiNet + """ + aliases = organization["alias"] + for alias in aliases: + county_code = alias.split("-")[0] + if self.sminet_service.client.is_supported_county_code(county_code): + return county_code + raise self.knm_service.NoSupportedCountyCodeFound( + "No supported county code found in alias list. Found: {}".format(aliases)) + + def create_sample_info(self, provider, sample, free_text): + sample_date_referral = provider.service_request["resource"]["authoredOn"] + # The date is on ISO8601 format: + sample_date_referral = dateutil.parser.isoparse(sample_date_referral) + + return SampleInfo(status=StatusType.FINAL_RESPONSE, + sample_id=sample.org_referral_code, + sample_date_arrival=sample.date_arrival, + sample_date_referral=sample_date_referral, + sample_material=sample.material, + sample_free_text_referral=free_text) + + def create_referring_clinic(self, provider): + """Creates a referring clinic object from KNM data""" + referring_clinic_name = provider.patient["managingOrganization"]["display"] + referring_clinic_county = self.get_county_from_organization( + provider.organization) + referring_doctor = provider.service_request["resource"]["requester"]["display"] + + # TODO: We don't have an address for the referring clinic + return ReferringClinic(referring_clinic_name, "", + referring_clinic_county, Doctor(referring_doctor)) + + def create_patient(self, provider): + """Creates a Patient object from KNM data""" + + def patient_identifier(): + try: + patient_identifier = provider.patient["identifier"] + except KeyError: + raise UnregisteredPatient( + "Missing field 'identifier' on the patient resource for {}".format(provider)) + + if len(patient_identifier) == 0: + raise UnregisteredPatient( + "Field 'identifier' is empty on patient resource for {}".format(provider)) + + try: + return patient_identifier[0]["value"] + except KeyError: + raise UnregisteredPatient( + "First entry in 'identifier' doesn't have a value key for {}".format(provider)) + + def patient_gender(): + gender = provider.patient["gender"] + try: + return self.MAP_GENDER_FROM_KNM_TO_SMINET[gender] + except KeyError: + raise IntegrationError( + "Unexpected value for key 'gender' ({}) for {}".format(gender, provider)) + + def patient_name(): + """ + The name of the patient. Returns an empty string if we don't have the expected data + """ + name = provider.patient["name"] + + if len(name) == 0: + return "" + + name = name[0] + + try: + return name["text"] + except KeyError: + return "" + + return Patient(patient_identifier(), + patient_gender(), + patient_name(), + None) + + def export_to_sminet(self, sample, doctor_name, lab_result, sample_free_text): + """ + Exports a SmiNetLabExport based on a sample + + :sample: A KNMSampleAccessor object + :doctor_name: Name of the doctor + :lab_result: A LabResult entry (for convenience one can use those created by SmiNetService) + """ + # Generate export: + provider = ServiceRequestProvider( + self.knm_service.client, sample.org_uri, sample.org_referral_code) + + sample_info = self.create_sample_info( + provider, sample, sample_free_text) + reporting_doctor = Doctor(doctor_name) + referring_clinic = self.create_referring_clinic(provider) + patient = self.create_patient(provider) + notification = Notification(sample_info, reporting_doctor, referring_clinic, patient, + lab_result) + laboratory = self.sminet_service.get_laboratory() + + export = SmiNetLabExport(datetime.now(), laboratory, notification) + + # Send it + self.sminet_service.client.create( + export, export.notification.sample_info.sample_id) + + +class IntegrationError(Exception): + pass + + +class UnregisteredPatient(IntegrationError): + pass diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/services/sminet_service.py b/clarity-ext-scripts/clarity_ext_scripts/covid/services/sminet_service.py new file mode 100644 index 0000000..736145b --- /dev/null +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/services/sminet_service.py @@ -0,0 +1,30 @@ +from sminet_client import (SmiNetConfig, LabResult, + Laboratory, LabDiagnosisType, SmiNetClient) + + +class SmiNetService(object): + + STATUS_SUCCESS = "success" # We've already uploaded the results to SmiNet + STATUS_ERROR = "error" # We've tried to upload the results to SmiNet but failed + + # The sample shouldn't be reported even if it's in the report step. This can be either + # because the extension figures this out based on business rules, or a research engineer + # marks it manually as such + STATUS_IGNORE = "ignore" + + STATUS_RETRY = "retry" + + def __init__(self, config): + self.config = SmiNetConfig(**config) + self.client = SmiNetClient(self.config) + + def get_laboratory(self): + """ + Retrieves the configured laboratory + """ + return Laboratory(self.config.lab_number, self.config.lab_name) + + @staticmethod + def create_scov2_positive_lab_result(): + lab_diagnosis = LabDiagnosisType("SCOV2", "SARS-CoV-2 (covid-19)") + return LabResult("C", lab_diagnosis) diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/utils.py b/clarity-ext-scripts/clarity_ext_scripts/covid/utils.py index b355bc4..d336dd6 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/utils.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/utils.py @@ -1,7 +1,7 @@ import re import time from datetime import datetime -from clarity_ext_scripts.covid.partner_api_client import PartnerAPIV7Client +from clarity_ext.domain import Sample, Artifact class UniqueBarcodeGenerator(object): @@ -9,7 +9,7 @@ class UniqueBarcodeGenerator(object): Generates a unique barcode ID that's guaranteed to be unique if there is only one person generating barcodes at a time (they use the current second to mark a unique time), while also being short enough to fit within a limited length barcode (currently 14 characters). - + Format: <2 digit type_id (0-99)><8 chars timestamp in hex><3 digits running> """ @@ -21,9 +21,9 @@ def __init__(self, prefix): self.pattern = re.compile( "^" + prefix + - "(?P\d{2})" + - "(?P\w{8})" + - "(?P\d{3})" + + r"(?P\d{2})" + + r"(?P\w{8})" + + r"(?P\d{3})" + "$") def parse(self, barcode): @@ -85,7 +85,6 @@ class CtmrCovidSubstanceInfo(object): STATUS_DISCARD = "DISCARD" STATUS_DISCARDED_AND_REPORTED = "DISCARDED_AND_REPORTED" - def __init__(self, substance): """ @@ -106,6 +105,13 @@ def _deduce_control_type_from_sample(sample): except AttributeError: return None + @property + def submitted_sample(self): + if isinstance(self.substance, Sample): + return self.substance + elif isinstance(self.substance, Artifact): + return self.substance.sample() + @staticmethod def _deduce_control_type_from_analyte_name(name): """ @@ -126,7 +132,8 @@ def control_type(self): if isinstance(self.substance, Sample): return self._deduce_control_type_from_sample(self.substance) elif isinstance(self.substance, Analyte): - control_type = self._deduce_control_type_from_analyte_name(self.substance.name) + control_type = self._deduce_control_type_from_analyte_name( + self.substance.name) if control_type: return control_type return self._deduce_control_type_from_sample(self.substance.sample()) @@ -134,16 +141,21 @@ def control_type(self): raise NotImplementedError("Not implemented substance type {}".format( type(self.substance))) + @property + def is_control(self): + return self.control_type is not None + @property + def sminet_status(self): + try: + return self.substance.udf_sminet_status + except AttributeError: + return None -def KNMClient(extension): - # A factory for a KnmClient from an extension - config = { - key: extension.config[key] - for key in [ - "test_partner_base_url", "test_partner_code_system_base_url", - "test_partner_user", "test_partner_password" - ] - } - return PartnerAPIV7Client(**config) + @sminet_status.setter + def sminet_status(self, value): + self.submitted_sample.udf_sminet_status = value + self.substance.udf_sminet_status = value + def __str__(self): + return "name={}, is_control={}".format(self.substance.name, self.is_control) diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/validate_discarded_samples.py b/clarity-ext-scripts/clarity_ext_scripts/covid/validate_discarded_samples.py index f954e7e..132762b 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/validate_discarded_samples.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/validate_discarded_samples.py @@ -2,7 +2,7 @@ from clarity_ext_scripts.covid.create_samples.common import ( BaseValidateRawSampleListExtension, BaseRawSampleListFile, BUTTON_TEXT_ASSIGN_UNREGISTERED_TO_ANONYMOUS) -from clarity_ext_scripts.covid.utils import KNMClient +from clarity_ext_scripts.covid.services.knm_service import KNMClientFromExtension class RawSampleListColumns(object): @@ -26,7 +26,7 @@ def execute(self): ordering_org = self.context.current_step.udf_ordering_organization except AttributeError: self.usage_error("You must select an ordering organization") - client = KNMClient(self) + client = KNMClientFromExtension(self) raw_sample_list = RawSampleListFile.create_from_context(self.context) @@ -56,7 +56,7 @@ def execute(self): self.context.file_service.upload( validated_sample_list.FILE_HANDLE, file_name, validated_sample_list_content, self.context.file_service.FILE_PREFIX_NONE) - if len(unregistered) > 0 : + if len(unregistered) > 0: self.usage_warning("The following sample are unregistered '{}'. Press '{}' " "to change the 'Status' to anonymous" "".format(unregistered, BUTTON_TEXT_ASSIGN_UNREGISTERED_TO_ANONYMOUS)) diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/validate_sample_creation_list.py b/clarity-ext-scripts/clarity_ext_scripts/covid/validate_sample_creation_list.py index 3302384..1a455e4 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/validate_sample_creation_list.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/validate_sample_creation_list.py @@ -1,9 +1,8 @@ import cStringIO import logging from datetime import datetime -from clarity_ext.extensions import GeneralExtension from clarity_ext_scripts.covid.controls import controls_barcode_generator -from clarity_ext_scripts.covid.utils import KNMClient +from clarity_ext_scripts.covid.services.knm_service import KNMClientFromExtension from clarity_ext_scripts.covid.create_samples.common import ( BaseRawSampleListFile, ValidatedSampleListFile, BaseValidateRawSampleListExtension, BUTTON_TEXT_ASSIGN_UNREGISTERED_TO_ANONYMOUS @@ -96,7 +95,7 @@ def execute(self): # 2. Create an API client # Make sure that there is a config at ~/.config/clarity-ext/clarity-ext.config - client = KNMClient(self) + client = KNMClientFromExtension(self) # 3. Read the raw sample list. raw_sample_list = RawSampleListFile.create_from_context(self.context) @@ -148,7 +147,7 @@ def execute(self): self.context.file_service.upload( "Validated sample list", file_name, validated_sample_list_content, self.context.file_service.FILE_PREFIX_NONE) - if len(unregistered) > 0 : + if len(unregistered) > 0: self.usage_warning("The following sample are unregistered '{}'. Press '{}' " "to change the 'Status' to anonymous" "".format(unregistered, BUTTON_TEXT_ASSIGN_UNREGISTERED_TO_ANONYMOUS)) diff --git a/clarity-ext-scripts/repos/clarity-ext b/clarity-ext-scripts/repos/clarity-ext index b3f6639..ad6cf90 160000 --- a/clarity-ext-scripts/repos/clarity-ext +++ b/clarity-ext-scripts/repos/clarity-ext @@ -1 +1 @@ -Subproject commit b3f66395e3bfbed3f712f1160b767cf750514781 +Subproject commit ad6cf90137aa1e1a5a71a56b37d9c275f9432c20 diff --git a/clarity-ext-scripts/requirements.txt b/clarity-ext-scripts/requirements.txt index 78c3b64..d069817 100644 --- a/clarity-ext-scripts/requirements.txt +++ b/clarity-ext-scripts/requirements.txt @@ -3,4 +3,5 @@ mock==3.0.5 pytest==4.6.9 requests[socks]==2.23.0 retry==0.9.2 -xlwt==1.3.0 +suds==0.4 +python-dateutil==2.8.1 diff --git a/clarity-ext-scripts/setup.sh b/clarity-ext-scripts/setup.sh index 101da66..e562190 100755 --- a/clarity-ext-scripts/setup.sh +++ b/clarity-ext-scripts/setup.sh @@ -11,3 +11,4 @@ pip install -e ./repos/clarity-ext pip install -r $SCRIPT_DIR/requirements.txt pip install -e $SCRIPT_DIR/. +pip install -e $SCRIPT_DIR/../sminet-client diff --git a/clarity-ext-scripts/tests/integration/test_knm_sminet.py b/clarity-ext-scripts/tests/integration/test_knm_sminet.py new file mode 100644 index 0000000..df54dd0 --- /dev/null +++ b/clarity-ext-scripts/tests/integration/test_knm_sminet.py @@ -0,0 +1,36 @@ +from datetime import datetime +from clarity_ext_scripts.covid.services.sminet_service import SmiNetService +from clarity_ext_scripts.covid.services.knm_sminet_service import KNMSmiNetIntegrationService +from clarity_ext_scripts.covid.partner_api_client import KARLSSON_AND_NOVAK, ORG_URI_BY_NAME +from clarity_ext.cli import load_config +from clarity_ext_scripts.covid.services.knm_service import KNMSampleAccessor, KNMService +from sminet_client import SampleMaterial + + +""" +Tests the SmiNet/KNM integration +""" + +config = load_config() + + +def test_can_create_request(): + """ + NOTE: We work against a whitelist on a jump server. Information available from team lead. + NOTE: Reads config from your clarity-ext config which should point to the test environment. + If sminet config is not available, the test is ignored. This is because the test + can't run on the build server because it's not whitelisted at the moment. + """ + + constant_date = datetime(2020, 4, 30) + knm_service = KNMService(config) + sminet_service = SmiNetService(config) + integration = KNMSmiNetIntegrationService( + config, knm_service, sminet_service) + org_uri = ORG_URI_BY_NAME[KARLSSON_AND_NOVAK] + sample = KNMSampleAccessor(org_uri, "5236417647", constant_date, SampleMaterial.SVALG) + sample_free_text = "Anamnes: Personalprov" + + lab_result = SmiNetService.create_scov2_positive_lab_result() + integration.export_to_sminet( + sample, "Lars Engstrand", lab_result, sample_free_text) diff --git a/clarity-ext-scripts/tests/integration/test_sminet.py b/clarity-ext-scripts/tests/integration/test_sminet.py deleted file mode 100644 index 7a7cd09..0000000 --- a/clarity-ext-scripts/tests/integration/test_sminet.py +++ /dev/null @@ -1,52 +0,0 @@ -import os -from uuid import uuid4 -import random -import string -import yaml -import pytest -from datetime import datetime -from clarity_ext_scripts.covid.sminet_client import (create_covid_request, SampleInfo, - ReferringClinic, Patient, Doctor, - SmiNetClient) - - -def not_setup_on_this_server(): - pytest.xfail( - "This test requires sminet to be setup in your clarity-ext config") - - -def get_sminet_client(): - path = os.path.expanduser("~/.config/clarity-ext/clarity-ext.config") - if not os.path.exists(path): - not_setup_on_this_server() - - with open(path) as f: - client_config = yaml.safe_load(f) - - if not "sminet_url" in client_config: - not_setup_on_this_server() - return SmiNetClient.SmiNetClientFromConfig(client_config) - - -def test_can_create_request(): - """ - NOTE: We work against a whitelist on a jump server. Information available from team lead. - NOTE: Reads config from your clarity-ext config which should point to the test environment. - If sminet config is not available, the test is ignored. This is because the test - can't run on the build server because it's not whitelisted at the moment. - """ - - client = get_sminet_client() - constant_date = datetime(2020, 4, 30) - request_created = datetime(2020, 5, 18, 18, 11, 6) - prefix = "int-tests-" - rnd = "".join(random.choice(string.ascii_uppercase + string.digits) - for _ in range(25 - len(prefix))) - sample_id = prefix + rnd - sample_info = SampleInfo(status=1, sample_id=sample_id, sample_date_arrival=constant_date, - sample_date_referral=constant_date, sample_material="Svalg", - sample_free_text_referral="Anamnes: Personalprov") - clinic = ReferringClinic("Clinic name", "", "C", Doctor("Some doctor")) - patient = Patient("1234", "k", "Some Name", 23) - reporting_doctor = Doctor("Lars Engstrand") - client.create(sample_info, clinic, patient, reporting_doctor) diff --git a/clarity-ext-scripts/tests/test_sminet_client.py b/clarity-ext-scripts/tests/test_sminet_client.py deleted file mode 100644 index 076c148..0000000 --- a/clarity-ext-scripts/tests/test_sminet_client.py +++ /dev/null @@ -1,28 +0,0 @@ -from datetime import datetime -from clarity_ext_scripts.covid.sminet_client import (create_covid_request, SampleInfo, - ReferringClinic, Patient, Doctor) -import lxml.etree as ET -import os - - -def test_can_create_expected_request(): - constant_date = datetime(2020, 4, 30) - request_created = datetime(2020, 5, 18, 18, 11, 6) - - sample_info = SampleInfo(status=1, sample_id="123", sample_date_arrival=constant_date, - sample_date_referral=constant_date, sample_material="Svalg", - sample_free_text_referral="Anamnes: Personalprov") - clinic = ReferringClinic("Clinic name", "", "C", Doctor("Some doctor")) - patient = Patient("1234", "k", "Some Name", 23) - reporting_doctor = Doctor("Lars Engstrand") - request = create_covid_request( - sample_info, clinic, patient, reporting_doctor, request_created) - - fixtures = os.path.join(os.path.dirname(__file__), "fixtures") - - # For debug reasons one can uncomment this: - # with open(os.path.join(fixtures, "sminet", "generated.xml"), "w") as fs: - # fs.write(request) - - with open(os.path.join(fixtures, "sminet", "valid_request.xml")) as fs: - assert fs.read() == request diff --git a/clarity-ext-scripts/tests/test_label_printer.py b/clarity-ext-scripts/tests/unit/test_label_printer.py similarity index 100% rename from clarity-ext-scripts/tests/test_label_printer.py rename to clarity-ext-scripts/tests/unit/test_label_printer.py diff --git a/clarity-ext-scripts/tests/test_partner_client.py b/clarity-ext-scripts/tests/unit/test_partner_client.py similarity index 100% rename from clarity-ext-scripts/tests/test_partner_client.py rename to clarity-ext-scripts/tests/unit/test_partner_client.py diff --git a/clarity-ext-scripts/tests/test_rtpcr_analysis_service.py b/clarity-ext-scripts/tests/unit/test_rtpcr_analysis_service.py similarity index 100% rename from clarity-ext-scripts/tests/test_rtpcr_analysis_service.py rename to clarity-ext-scripts/tests/unit/test_rtpcr_analysis_service.py diff --git a/sminet-client/README.md b/sminet-client/README.md new file mode 100644 index 0000000..478d752 --- /dev/null +++ b/sminet-client/README.md @@ -0,0 +1,8 @@ +# SmiNet client + +This package contains a client for SmiNet, Public Health Agency of Sweden's web service for reporting laboratory results. + +## Installation + +pip install -e . + diff --git a/sminet-client/setup.py b/sminet-client/setup.py new file mode 100644 index 0000000..e8a2721 --- /dev/null +++ b/sminet-client/setup.py @@ -0,0 +1,23 @@ +from setuptools import find_packages, setup + +setup( + name='sminet-client', + version='1.0.0', + packages=find_packages(exclude=['tests']), + install_requires=['suds', 'pyyaml', 'lxml'], + zip_safe=False, + platforms='any', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', + 'Operating System :: POSIX', + 'Operating System :: MacOS', + 'Operating System :: Unix', + 'Operating System :: Windows', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Topic :: Software Development :: Libraries :: Python Modules', + ] +) diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/sminet_client.py b/sminet-client/sminet_client/__init__.py similarity index 58% rename from clarity-ext-scripts/clarity_ext_scripts/covid/sminet_client.py rename to sminet-client/sminet_client/__init__.py index 9bffb2d..a3f8f1d 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/sminet_client.py +++ b/sminet-client/sminet_client/__init__.py @@ -1,15 +1,21 @@ # -*- coding: utf-8 -*- -import datetime -import requests +import os +import yaml import logging +import codecs +import base64 +import lxml +from lxml import objectify import lxml.etree as ET +from suds.client import Client +from suds.xsd.doctor import ImportDoctor, Import logger = logging.getLogger(__name__) """ -A Python client for the SmiNet site +A client for the SmiNet site NOTE: Some of the documentation is in Swedish. In those cases, it has been copied directly from @@ -27,7 +33,7 @@ def add_child(parent, elementName, value, validator=None): child_element = ET.SubElement(parent, elementName) if validator: value = validator(value) - child_element.text = str(value) # Must always be cast to string + child_element.text = unicode(value) # Must always be cast to string #### @@ -50,11 +56,19 @@ def IdType(python_string): return python_string +class Gender(object): + MALE = "m" + FEMALE = "k" + UNKNOWN = "o" + + ALL = [MALE, FEMALE, UNKNOWN] + + def SexType(python_string): """ Avser patientens kön ('o'/'O' = Okänt). """ - if python_string.lower() not in ["m", "k", "o"]: + if python_string.lower() not in Gender.ALL: raise SmiNetValidationError( "Unknown sex type {}".format(python_string)) return python_string @@ -71,13 +85,64 @@ def DiagnosticMethod(python_string): return python_string +COUNTY_STOCKHOLM = "Stockholm" +COUNTY_UPPSALA = "Uppsala" +COUNTY_SODERMANLAND = "Sodermanland" +COUNTY_OSTERGOTLAND = "Ostergotland" +COUNTY_JONKOPING = "Jonkoping" +COUNTY_KRONOBERG = "Kronoberg" +COUNTY_KALMAR = "Kalmar" +COUNTY_GOTLAND = "Gotland" +COUNTY_BLEKINGE = "Blekinge" +COUNTY_SKANE = "Skane" +COUNTY_HALLAND = "Halland" +COUNTY_VASTRAGOTALAND = "Vastragotaland" +COUNTY_VARMLAND = "Varmland" +COUNTY_OREBRO = "Orebro" +COUNTY_VASTMANLAND = "Vastmanland" +COUNTY_DALARNA = "Dalarna" +COUNTY_GAVLEBORG = "Gavleborg" +COUNTY_VASTERNORRLAND = "Vasternorrland" +COUNTY_JAMTLAND = "Jamtland" +COUNTY_VASTERBOTTEN = "Vasterbotten" +COUNTY_NORRBOTTEN = "Norrbotten" + + +MAP_COUNTY_CODE_TO_NAME = { + "AB": COUNTY_STOCKHOLM, + "C": COUNTY_UPPSALA, + "D": COUNTY_SODERMANLAND, + "E": COUNTY_OSTERGOTLAND, + "F": COUNTY_JONKOPING, + "G": COUNTY_KRONOBERG, + "H": COUNTY_KALMAR, + "I": COUNTY_GOTLAND, + "K": COUNTY_BLEKINGE, + "M": COUNTY_SKANE, + "N": COUNTY_HALLAND, + "O": COUNTY_VASTRAGOTALAND, + "S": COUNTY_VARMLAND, + "T": COUNTY_OREBRO, + "U": COUNTY_VASTMANLAND, + "W": COUNTY_DALARNA, + "X": COUNTY_GAVLEBORG, + "Y": COUNTY_VASTERNORRLAND, + "Z": COUNTY_JAMTLAND, + "AC": COUNTY_VASTERBOTTEN, + "BD": COUNTY_NORRBOTTEN, + + "OB": COUNTY_VASTRAGOTALAND, + "OG": COUNTY_VASTRAGOTALAND, + "OS": COUNTY_VASTRAGOTALAND, + "OT": COUNTY_VASTRAGOTALAND, +} + + def CountyType(python_string): """ Avser smittskyddsläkarens landstingsbokstav. """ - if python_string not in ["AB", "C", "D", "E", "F", "G", "H", "I", "K", "M", - "N", "O", "OB", "OG", "OS", "OT", "S", "T", "U", "W", - "X", "Y", "Z", "AC", "BD"]: + if python_string not in MAP_COUNTY_CODE_TO_NAME.keys(): raise SmiNetValidationError( "Unknown county code {}".format(python_string)) return python_string @@ -171,57 +236,91 @@ def Version(version_number): return element -def Laboratory(lab_number, lab_name): - """ - :lab_number: Non-negative integer. Each lab that connects with "automatisk överföring" against - SmiNet gets a unique identification number - :lab_name: Name of the laboratory exporting the file. Max 255 characters. - """ - laboratory = ET.Element("laboratory") - add_child(laboratory, "labNumber", lab_number, NonNegativeInteger) - add_child(laboratory, 'labName', lab_name, LongLimitedString) - - return laboratory - - # NOTE: These are our translations of the status in the xsd StatusType documentation, which is in # Swedish. Might be inaccurate. -STATUS_FINAL_RESPONSE = 1 -STATUS_COMPLEMENTARY_DATA_PENDING = 2 -STATUS_REVOCATION_OF_PREVIOUS_REPORT = 3 -STATUS_COMPLEMENTARY_DATA = 4 -def StatusType(status): +class StatusType(object): """ Original: Avser en anmälans status. Förklaring: 1. Slutsvar. 2. Komplettering kommer. 3. Makulering av en tidigare anmälan. 4. Komplettering av en tidigare anmälan. """ - if status not in [STATUS_FINAL_RESPONSE, - STATUS_COMPLEMENTARY_DATA_PENDING, - STATUS_REVOCATION_OF_PREVIOUS_REPORT, - STATUS_COMPLEMENTARY_DATA]: - raise SmiNetValidationError("The status {} is not in the list of accepted statuses" - .format(status)) - return status + FINAL_RESPONSE = 1 + COMPLEMENTARY_DATA_PENDING = 2 + REVOCATION_OF_PREVIOUS_REPORT = 3 + COMPLEMENTARY_DATA = 4 + + ALL = [FINAL_RESPONSE, + COMPLEMENTARY_DATA_PENDING, + REVOCATION_OF_PREVIOUS_REPORT, + COMPLEMENTARY_DATA] + + def __init__(self, status): + if status not in self.ALL: + raise SmiNetValidationError("The status {} is not in the list of accepted statuses" + .format(status)) + self.value = status -def SampleMaterialType(material_type): + +class SampleMaterial(object): """ Avser undersökningsmatrial. (Notera att det är tillåtet att byta ut 'å' och 'ä' mot 'a' samt att 'ö' kan bytas ut mot 'o'. Detta alternativ finns om det skulle bli problem med valideringen pga konstigheter i teckentabeller.) """ - supported = ["Annat" "Bio", "Blod", "Bronk", "Feces", "Likv", "Lymf", - "Nfary", "Nasa", "Pleur", "Sekr", "Serum", "Sput", "Svalg", "Sar", - "Urin", "VSK", "Led", "Perik", "Fost", "Asci", "Melor", "Uret", - "Rect", "Cerv", "VagSek", "Asp", "Blodsr", "Cervur", "Infart", "Nsp", - "Perin", "Poolat", "Saliv", "Vagur", "Ogon", "Oron"] - if material_type not in supported: - raise SmiNetValidationError( - "Material type not supported: {}".format(material_type)) - return material_type + OTHER = "Annat" + BIO = "Bio" + BLOD = "Blod" + BRONK = "Bronk" + FECES = "Feces" + LIKV = "Likv" + LYMF = "Lymf" + NFARY = "Nfary" + NASA = "Nasa" + PLEUR = "Pleur" + SEKR = "Sekr" + SERUM = "Serum" + SPUT = "Sput" + SVALG = "Svalg" + SAR = "Sar" + URIN = "Urin" + VSK = "VSK" + LED = "Led" + PERIK = "Perik" + FOST = "Fost" + ASCI = "Asci" + MELOR = "Melor" + URET = "Uret" + RECT = "Rect" + CERV = "Cerv" + VAGSEK = "VagSek" + ASP = "Asp" + BLODSR = "Blodsr" + CERVUR = "Cervur" + INFART = "Infart" + NSP = "Nsp" + PERIN = "Perin" + POOLAT = "Poolat" + SALIV = "Saliv" + VAGUR = "Vagur" + OGON = "Ogon" + ORON = "Oron" + + ALL = [OTHER, BIO, BLOD, BRONK, FECES, LIKV, + LYMF, NFARY, NASA, PLEUR, SEKR, SERUM, + SPUT, SVALG, SAR, URIN, VSK, LED, + PERIK, FOST, ASCI, MELOR, URET, RECT, + CERV, VAGSEK, ASP, BLODSR, CERVUR, INFART, + NSP, PERIN, POOLAT, SALIV, VAGUR, + OGON, ORON] + + def __init__(self, value): + if value not in self.ALL: + raise SmiNetValidationError( + "Material type not supported: {}".format(value)) + self.value = value ##### @@ -236,6 +335,23 @@ def __str__(self): return ET.tostring(self.to_element(), pretty_print=True) +class Laboratory(SmiNetComplexType): + def __init__(self, lab_number, lab_name): + """ + :lab_number: Non-negative integer. Each lab that connects with "automatisk överföring" against + SmiNet gets a unique identification number + :lab_name: Name of the laboratory exporting the file. Max 255 characters. + """ + self.lab_number = NonNegativeInteger(lab_number) + self.lab_name = LongLimitedString(lab_name) + + def to_element(self, element_name="laboratory"): + laboratory = ET.Element(element_name) + add_child(laboratory, "labNumber", self.lab_number) + add_child(laboratory, 'labName', self.lab_name) + return laboratory + + class ReferringClinic(SmiNetComplexType): def __init__(self, referring_clinic_name, referring_clinic_address, referring_clinic_county, referring_doctor=None): @@ -307,7 +423,7 @@ def __init__(self, # Is a string according to the docs self.sample_id = ShortString(str(sample_id)) self.sample_date_arrival = SmiNetDate(sample_date_arrival) - self.sample_material = SampleMaterialType(sample_material) + self.sample_material = SampleMaterial(sample_material) self.optional_reference = ShortLimitedString( optional_reference) if optional_reference else None self.sample_date_referral = SmiNetDate( @@ -319,11 +435,11 @@ def __init__(self, def to_element(self, element_name="sampleInfo"): element = ET.Element(element_name) - add_child(element, "status", self.status) + add_child(element, "status", self.status.value) add_child(element, "sampleNumber", self.sample_id) add_child(element, "sampleDateArrival", self.sample_date_arrival) add_child(element, "sampleDateReferral", self.sample_date_referral) - add_child(element, "sampleMaterial", self.sample_material) + add_child(element, "sampleMaterial", self.sample_material.value) add_child(element, "optionalReference", self.optional_reference) add_child(element, "sampleFreeTextLab", self.sample_free_text_lab) add_child(element, "sampleFreeTextReferral", @@ -331,7 +447,39 @@ def to_element(self, element_name="sampleInfo"): return element -class NotificationType(SmiNetComplexType): +class Patient(SmiNetComplexType): + """ + Information för att identifiera patienten. + """ + + def __init__(self, patient_id, patient_sex, patient_name=None, patient_age=None): + """ + :patient_id: Tillåtna typer är personnummer, rikskod och reservkod + :patient_sex: Patientens kön + :patient_name: Patientens namn + :patient_age: Patientens ålder + """ + self.patient_id = IdType(patient_id) + self.patient_sex = SexType(patient_sex) + self.patient_name = LimitedString(patient_name) + self.patient_age = NonNegativeInteger( + patient_age) if patient_age else None + + def to_element(self, element_name="patient"): + element = ET.Element(element_name) + add_child(element, "patientId", self.patient_id) + add_child(element, "patientSex", self.patient_sex) + add_child(element, "patientName", self.patient_name) + add_child(element, "patientAge", self.patient_age) + return element + + +class Notification(SmiNetComplexType): + + SampleInfo = SampleInfo + Doctor = Doctor + ReferringClinic = ReferringClinic + Patient = Patient def __init__(self, sample_info, reporting_doctor, referring_clinic, patient, lab_result): """ @@ -395,9 +543,9 @@ class LabResult(SmiNetComplexType): def __init__(self, diagnostic_method, lab_diagnosis): """ :diagnostic_method: Observera att det är SmiNet-koden för diagnostisk metod som - ska användas. + ska användas. :lab_diagnosis: Viktig typningsinformation om provet, som till exempel eventuell - typningsinformation om sådan finns. + typningsinformation om sådan finns. """ self.diagnostic_method = DiagnosticMethod(diagnostic_method) @@ -411,34 +559,7 @@ def to_element(self, element_name="labResult"): return element -class Patient(SmiNetComplexType): - """ - Information för att identifiera patienten. - """ - - def __init__(self, patient_id, patient_sex, patient_name=None, patient_age=None): - """ - :patient_id: Tillåtna typer är personnummer, rikskod och reservkod - :patient_sex: Patientens kön - :patient_name: Patientens namn - :patient_age: Patientens ålder - """ - self.patient_id = IdType(patient_id) - self.patient_sex = SexType(patient_sex) - self.patient_name = LimitedString(patient_name) - self.patient_age = NonNegativeInteger( - patient_age) if patient_age else None - - def to_element(self, element_name="patient"): - element = ET.Element(element_name) - add_child(element, "patientId", self.patient_id) - add_child(element, "patientSex", self.patient_sex) - add_child(element, "patientName", self.patient_name) - add_child(element, "patientAge", self.patient_age) - return element - - -def SmiNetLabExport(created, lab_number, notification): +class SmiNetLabExport(): """ Creates a validated lab export as XML that is acceptable for the SmiNet endpoint. @@ -447,106 +568,185 @@ def SmiNetLabExport(created, lab_number, notification): Original documentation in xsd: Rootelementet i xml-dokumentet. Varje exportfil (XML-fil) måste innehålla ett och endast ett sådant element. """ - element = ET.Element('smiNetLabExport') # TODO: xlmns and xsi - - # # smiNetLabExport/version/version-number - element.append(Version("4.0.0")) - # # smiNetLabExport/dateTimeCreated - date_time_created = ET.Element("dateTimeCreated") - date_time_created.text = SmiNetDateTime(created) - element.append(date_time_created) + # For convenience, all the types that make up an export are defined here: + Laboratory = Laboratory + Notification = Notification - # # smiNetLabExport/laboratory - element.append(Laboratory(lab_number, "National Pandemic Center at KI")) - - element.append(notification.to_element()) - return element + def __init__(self, created, laboratory, notification): + self.created = created + self.laboratory = laboratory + self.notification = notification + self.version = Version("4.0.0") + def to_element(self, xsd_url): + element = ET.Element('smiNetLabExport') -def create_scov2_positive_lab_result(): - lab_diagnosis = LabDiagnosisType("SCOV2", "SARS-CoV-2 (covid-19)") - return LabResult("C", lab_diagnosis) + # element.attrib['xmlns:xsi'] = "" + element.attrib["{http://www.w3.org/2001/XMLSchema-instance}noNamespaceSchemaLocation"] = xsd_url + # # smiNetLabExport/version/version-number + element.append(self.version) -def create_covid_request(sample_info, referring_clinic, patient, reporting_doctor, created=None): - """ - Creates a request to SmiNet that's valid for the Covid project. - - :sample_info: An object created with the SampleInfo factory - :referring_clinic: An object created with the ReferringClinic factory - :patient: An object created with the Patient factory - """ + # # smiNetLabExport/dateTimeCreated + date_time_created = ET.Element("dateTimeCreated") + date_time_created.text = SmiNetDateTime(self.created) + element.append(date_time_created) - if not created: - created = datetime.datetime.now() + # # smiNetLabExport/laboratory + element.append(self.laboratory.to_element()) + element.append(self.notification.to_element()) + return element - notification = NotificationType(sample_info, reporting_doctor, referring_clinic, patient, - create_scov2_positive_lab_result()) + def to_document(self, xsd_url): + """ + Creates an XML string representation that should be accepted by SmiNet + """ + export = self.to_element(xsd_url) + return ET.tostring(export, pretty_print=True, encoding="ISO-8859-1") - lab_number = 91 # TODO, ensure that this is correct - contract = SmiNetLabExport(created, lab_number, notification) - return ET.tostring(contract, pretty_print=True, encoding="ISO-8859-1") +class SmiNetError(Exception): + pass -class SmiNetValidationError(Exception): +class SmiNetValidationError(SmiNetError): pass -class SmiNetRequestError(Exception): +class SmiNetRequestError(SmiNetError): pass class SmiNetClient(object): - def __init__(self, sminet_url, sminet_username, sminet_password, sminet_proxy=None): + + SMINET_ENVIRONMENT_STAGE = "stage" + SMINET_ENVIRONMENT_PROD = "prod" + + def __init__(self, config): """ - :sminet_url: The url - :sminet_username: The username - :sminet_password: The password - :proxy: Proxy info if the client requires socks (we access SmiNet via a jump host in dev) + :config: A configuration object of type SmiNetConfig """ - self.url = sminet_url - if not "https://" in self.url: - raise AssertionError("The SmiNet url should use https") - self.username = sminet_username - self.password = sminet_password - self.proxy = sminet_proxy - def create(self, sample_info, referring_clinic, patient, reporting_doctor, created=None): + self.config = config + self._soap_client = None + + if config.environment == self.SMINET_ENVIRONMENT_STAGE: + self.url = "https://stage.sminet.se/sminetlabxmlzone/services/SmiNetLabXmlZone" + self.xsd_url = "http://stage.sminet.se/xml-schemas/SmiNetLabExport.xsd" + self.wsdl_url = "https://stage.sminet.se/sminetlabxmlzone/services/SmiNetLabXmlZone?wsdl" + elif config.environment == self.SMINET_ENVIRONMENT_PROD: + self.url = "https://sminet.se/sminetlabxmlzone/services/SmiNetLabXmlZone" + self.xsd_url = "http://sminet.se/xml-schemas/SmiNetLabExport.xsd" + self.wsdl_url = "https://sminet.se/sminetlabxmlzone/services/SmiNetLabXmlZone?wsdl" + + @property + def soap_client(self): + if self._soap_client is None: + imp = Import('http://schemas.xmlsoap.org/soap/encoding/') + doctor = ImportDoctor(imp) + self._soap_client = Client(self.wsdl_url, + doctor=doctor) + return self._soap_client + + def is_supported_county_code(self, county_code): + return county_code in MAP_COUNTY_CODE_TO_NAME.keys() + + def create(self, sminet_lab_export, file_name): """ Creates the entry in SmiNet - :sample_info: A SampleInfo instance - :referring_clinic: A ReferringClinic instance - :patient: A Patient instance - :reporting_doctor: A Doctor instance - :created: The date when the request is created. Defaults to now() + :export: An instance of SmiNetLabExport + :file_name: Name of the file in SmiNet's database """ logger.info("Creating a request at SmiNet") - headers = {"Content-Type": "application/xml"} - data = create_covid_request( - sample_info, referring_clinic, patient, reporting_doctor, created) - response = requests.post(self.url, data=data, proxies=dict( - https=self.proxy), headers=headers) + doc = sminet_lab_export.to_document(self.xsd_url) + self._send_file(doc, file_name) + + def _parse_xml(self, xml): + """ + Returns a Python object from an XML response string" + """ + xml = codecs.encode(xml, "utf-8") + obj = objectify.fromstring(xml) + return obj - if response.status_code != requests.codes.created: - raise SmiNetRequestError(response.text) + def _send_file(self, xml_file, file_name): + """ + Sends an XML document to SmiNet. - logger.info("Request created at SmiNet. Response code was={}".format( - response.status_code)) + Raises a SmiNetRequestError if the return code is not zero. - @classmethod - def SmiNetClientFromConfig(cls, extension_config): + :xml_file: A valid xml document as a string. Use SmiNetLabExport to generate + a valid document. + :file_name: The name of the file in SmiNet + """ + base64encoded = base64.b64encode(xml_file) + + resp = self.soap_client.service.submitFile( + self.config.username, self.config.password, file_name, base64encoded) + resp = self._parse_xml(resp) + + if resp.returnCode != 0: + raise SmiNetRequestError(resp.message) + + def validate(self, xml): + """ + Validates an XML contract against the xsd SmiNet provides + """ + xml_validator = lxml.etree.XMLSchema( + file="http://stage.sminet.se/xml-schemas/SmiNetLabExport.xsd") + is_valid = xml_validator.validate(xml) + + if not is_valid: + raise SmiNetValidationError( + "Not able to validate xml against the xsd file") + + +class SmiNetConfig(object): + DEFAULT_PATH_ETC = "/etc/sminet_client/sminet_client.config" + DEFAULT_PATH_USER = "~/.config/sminet_client/sminet_client.config" + + def __init__(self, sminet_username, + sminet_password, + sminet_proxy, + sminet_environment, + sminet_lab_name, + sminet_lab_number, + **kwargs): """ - Creates a SmiNetClient from a clarity-ext extension config (self.config) + Creates a configuration file that's valid for this SmiNet client. + + Searches the search paths for a valid configuration file + + :sminet_username: The username + :sminet_password: The password + :sminet_environment: The environment, e.g. SmiNetClient.SMINET_ENVIRONMENT_STAGE + :proxy: Proxy info if the service is only accessible via a proxy + :lab_name: The name of your lab + :lab_number: The name of your lab """ - filtered = { - key: extension_config[key] - for key in [ - "sminet_url", - "sminet_username", - "sminet_password", - "sminet_proxy"] if key in extension_config - } - return cls(**filtered) + self.username = sminet_username + self.password = sminet_password + self.proxy = sminet_proxy + self.environment = sminet_environment + self.lab_name = sminet_lab_name + self.lab_number = sminet_lab_number + + @classmethod + def create_from_search_paths(cls, paths=None): + if not paths: + paths = [cls.DEFAULT_PATH_ETC, cls.DEFAULT_PATH_USER] + + for path in paths: + path = os.path.expanduser(path) + if os.path.exists(path): + with open(path, "r") as f: + config = yaml.safe_load(f) + return cls(**config) + raise SmiNetConfigNotFoundError( + "No config file found in {}".format(paths)) + + +class SmiNetConfigNotFoundError(Exception): + pass diff --git a/clarity-ext-scripts/tests/fixtures/sminet/valid_request.xml b/sminet-client/tests/fixtures/valid_request.xml similarity index 74% rename from clarity-ext-scripts/tests/fixtures/sminet/valid_request.xml rename to sminet-client/tests/fixtures/valid_request.xml index 7709741..762f07f 100644 --- a/clarity-ext-scripts/tests/fixtures/sminet/valid_request.xml +++ b/sminet-client/tests/fixtures/valid_request.xml @@ -1,12 +1,12 @@ - + 4.0.0 2020-05-18 18:11:06 - 91 - National Pandemic Center at KI + 1 + Some lab @@ -15,23 +15,23 @@ 2020-04-30 2020-04-30 Svalg - Anamnes: Personalprov + Free text - Lars Engstrand + Reporting Doctor Clinic name C - Some doctor + Referring doctor 1234 k - Some Name + Patient Name 23 diff --git a/sminet-client/tests/integration/test_sminet_client.py b/sminet-client/tests/integration/test_sminet_client.py new file mode 100644 index 0000000..d6c2b67 --- /dev/null +++ b/sminet-client/tests/integration/test_sminet_client.py @@ -0,0 +1,96 @@ +import string +import random +from datetime import datetime +import lxml.etree as ET +import pytest +from sminet_client import (SampleInfo, ReferringClinic, Patient, Doctor, SmiNetConfig, + LabDiagnosisType, LabResult, Laboratory, SmiNetLabExport, + Notification, SampleMaterial, SmiNetClient, SmiNetValidationError) + + +""" +Tests an integration to the SmiNet. Requires a configuration file (see README.md) +""" + +config = SmiNetConfig.create_from_search_paths() + + +def generate_valid_contract_with_random_sample_id(): + """ + Returns a tuple of the parameters required to upload sample info via client.create + """ + + constant_date = datetime(2020, 4, 30) + request_created = datetime(2020, 5, 18, 18, 11, 6) + prefix = "int-tests-" + rnd = "".join(random.choice(string.ascii_uppercase + string.digits) + for _ in range(25 - len(prefix))) + sample_id = prefix + rnd + + sample_info = SampleInfo(status=1, + sample_id=sample_id, + sample_date_arrival=constant_date, + sample_date_referral=constant_date, + sample_material=SampleMaterial.SVALG, + sample_free_text_referral="Extra info") + referring_clinic = ReferringClinic( + "Clinic name", "", "C", Doctor("Some doctor")) + patient = Patient("121212-1212", "k", "Tolvan Tolvansson", 23) + reporting_doctor = Doctor("Reporting Doctor") + + lab_diagnosis = LabDiagnosisType("SCOV2", "SARS-CoV-2 (covid-19)") + lab_result = LabResult("C", lab_diagnosis) + notification = Notification(sample_info, reporting_doctor, referring_clinic, patient, + lab_result) + laboratory = Laboratory(config.lab_number, config.lab_name) + export = SmiNetLabExport(request_created, laboratory, notification) + return export + + +def test_can_create_request(): + """ + Tests creating a request. + """ + client = SmiNetClient(config) + export = generate_valid_contract_with_random_sample_id() + timestamp = datetime.now().strftime("%y%m%dT%H%M%S") + file_name = "{}-{}".format(timestamp, + export.notification.sample_info.sample_id) + client.create(export, file_name) + + +def test_can_create_request_with_missing_info(): + """ + Ensure that we can create info were the following data is missing + """ + client = SmiNetClient(config) + export = generate_valid_contract_with_random_sample_id() + + export.notification.patient.patient_age = "1" # This can not be empty + export.notification.patient.patient_age = None + export.notification.patient.patient_name = "" + + timestamp = datetime.now().strftime("%y%m%dT%H%M%S") + file_name = "some-empty-{}-{}".format(timestamp, + export.notification.sample_info.sample_id) + client.create(export, file_name) + + +def test_covid_request_xml_is_valid_against_xsd_schema(): + client = SmiNetClient(config) + export = generate_valid_contract_with_random_sample_id() + xml = export.to_element(client.SMINET_ENVIRONMENT_STAGE) + client.validate(xml) + + +def test_covid_request_xml_is_not_valid_against_xsd_schema(): + """ + Ensure that we get a validation error if making a significant change to the xml doc + """ + client = SmiNetClient(config) + export = generate_valid_contract_with_random_sample_id() + xml = export.to_element(client.SMINET_ENVIRONMENT_STAGE) + element = ET.Element("extra") + xml.append(element) + with pytest.raises(SmiNetValidationError): + client.validate(xml) diff --git a/sminet-client/tests/unit/__init__.py b/sminet-client/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sminet-client/tests/unit/test_sminet_client.py b/sminet-client/tests/unit/test_sminet_client.py new file mode 100644 index 0000000..148d630 --- /dev/null +++ b/sminet-client/tests/unit/test_sminet_client.py @@ -0,0 +1,41 @@ +import os +from datetime import datetime +from sminet_client import (Doctor, Notification, LabDiagnosisType, + LabResult, Laboratory, SmiNetLabExport, StatusType, SampleMaterial) + + +def get_fixture(fname): + fixtures = os.path.join(os.path.dirname(__file__), "..", "fixtures") + with open(os.path.join(fixtures, fname)) as fs: + return fs.read() + + +def create_document(): + constant_date = datetime(2020, 4, 30) + request_created = datetime(2020, 5, 18, 18, 11, 6) + + sample_info = Notification.SampleInfo( + status=StatusType.FINAL_RESPONSE, + sample_id="123", + sample_date_arrival=constant_date, + sample_date_referral=constant_date, + sample_material=SampleMaterial.SVALG, + sample_free_text_referral="Free text") + referring_clinic = Notification.ReferringClinic( + "Clinic name", "", "C", Doctor("Referring doctor")) + patient = Notification.Patient("1234", "k", "Patient Name", 23) + reporting_doctor = Notification.Doctor("Reporting Doctor") + + lab_diagnosis = LabDiagnosisType("SCOV2", "SARS-CoV-2 (covid-19)") + lab_result = LabResult("C", lab_diagnosis) + notification = Notification(sample_info, reporting_doctor, referring_clinic, patient, + lab_result) + laboratory = Laboratory(1, "Some lab") + export = SmiNetLabExport(request_created, laboratory, notification) + return export.to_document("http://stage.sminet.se/xml-schemas/SmiNetLabExport.xsd") + + +def test_can_create_expected_request(): + expected = get_fixture("valid_request.xml") + actual = create_document() + assert expected == actual