Merge pull request #155 from ctmrbio/dev

New deployment

.github/workflows/pythonapp.yml

@@ -19,10 +19,21 @@ jobs:
         uses: actions/setup-python@v1
         with:
           python-version: 2.7
+      - name: Add dummy genologicsrc
+        run: |
+          cat << EOF > ~/.genologicsrc
+          [genologics] 
+          BASEURI=https://fancy.server
+          USERNAME=username
+          PASSWORD=pass
+          EOF
+          cat ~/.genologicsrc
       - name: Install dependencies
         run: |
           ./clarity-ext-scripts/setup.sh
       - name: Test with pytest
         run: |
           pip install pytest
-          pytest ./clarity-ext-scripts/tests
+          pytest ./clarity-ext-scripts/tests/unit
+          pytest ./sminet-client/tests/unit
+

README.md

@@ -68,5 +68,9 @@ The [clarity-ext](https://github.com/molmed/clarity-ext) package is designed to
 
 Extensions that use this framework are all in ./clarity-ext-scripts. Refer to the [README](./clarity-ext-scripts/README.md) for more information on this package.
 
+## SmiNet 
+
+There is a client for SmiNet integration in ./sminet_client/. Refer to the [README](./sminet_client/README.md) for more information on the package.
+
 ## More information
 More information about the LIMS system including details on the Miniconda installation and starting the server may be found [here](https://github.com/ctmrbio/wiki/wiki/CTMR-LIMS-(PROD-and-STAGE)).

clarity-ext-scripts/clarity_ext_scripts/covid/create_samples/assign_unregistered_to_anonymous.py

@@ -1,17 +1,17 @@
 from uuid import uuid4
 from clarity_ext.extensions import GeneralExtension
 from clarity_ext_scripts.covid.create_samples.common import ValidatedSampleListFile
-from clarity_ext_scripts.covid.utils import KNMClient
+from clarity_ext_scripts.covid.services.knm_service import KNMClientFromExtension
 from clarity_ext_scripts.covid.partner_api_client import (
-        ORG_URI_BY_NAME, TESTING_ORG, CouldNotCreateServiceRequest, ServiceRequestAlreadyExists)
+    ORG_URI_BY_NAME, TESTING_ORG, CouldNotCreateServiceRequest, ServiceRequestAlreadyExists)
 
 
 class Extension(GeneralExtension):
     """
     Goes through the 'Validated sample list' ensuring that each sample that has the unregistered
     status gets an anonymous service request id.
 
-    Uploads a new file to the validated sample list file handle. The name of it will be on the form 
+    Uploads a new file to the validated sample list file handle. The name of it will be on the form
     'validated_sample_list_no_unregistered_<timestamp>.csv'
 
     Note that the user is not required to run this extension. They can create samples directly
@@ -20,7 +20,7 @@ class Extension(GeneralExtension):
     """
 
     def execute(self):
-        client = KNMClient(self)
+        client = KNMClientFromExtension(self)
         validated_sample_list = ValidatedSampleListFile.create_from_context(
             self.context)
         no_unregistered = ValidatedSampleListFile(validated_sample_list.csv)

clarity-ext-scripts/clarity_ext_scripts/covid/discard_samples.py

@@ -1,12 +1,9 @@
 import logging
 from datetime import datetime
-from clarity_ext.extensions import GeneralExtension
 from clarity_ext_scripts.covid.partner_api_client import (
-    PartnerAPIV7Client, TESTING_ORG, ORG_URI_BY_NAME, COVID_RESPONSE_FAILED,
-    PartnerClientAPIException)
-from clarity_ext_scripts.covid.rtpcr_analysis_service import FAILED_STATES
-from clarity_ext_scripts.covid.controls import Controls
-from clarity_ext_scripts.covid.utils import CtmrCovidSubstanceInfo, KNMClient
+        TESTING_ORG, ORG_URI_BY_NAME, COVID_RESPONSE_FAILED,
+        PartnerClientAPIException)
+from clarity_ext_scripts.covid.utils import CtmrCovidSubstanceInfo, KNMClientFromExtension
 from clarity_ext_scripts.covid.import_samples import BaseCreateSamplesExtension
 
 logger = logging.getLogger(__name__)
@@ -61,18 +58,20 @@ def report(self, analyte):
         self.context.commit()
 
     def execute(self):
-        self.client = KNMClient(self)
+        self.client = KNMClientFromExtension(self)
         for plate in self.context.input_containers:
             for well in plate.occupied:
                 already_uploaded = False
                 try:
                     already_uploaded = well.artifact.udf_knm_result_uploaded == UDF_TRUE
+
                 except AttributeError:
                     pass
 
                 if already_uploaded:
                     logger.info("Analyte {} has already been uploaded".format(
                         well.artifact.name))
+
                     continue
 
                 self.report(well.artifact)

clarity-ext-scripts/clarity_ext_scripts/covid/import_samples.py

@@ -1,6 +1,6 @@
 from clarity_ext.domain import Container, Sample
 from clarity_ext_scripts.covid.controls import controls_barcode_generator, Controls
-from clarity_ext_scripts.covid.utils import KNMClient
+from clarity_ext_scripts.covid.services.knm_service import KNMClientFromExtension
 from clarity_ext_scripts.covid.create_samples.common import BaseCreateSamplesExtension
 
 
@@ -118,7 +118,7 @@ def execute(self):
         """
         Creates samples from a validated import file
         """
-        self.client = KNMClient(self)
+        self.client = KNMClientFromExtension(self)
 
         # This is for debug reasons only. Set this to True to create samples even if they have
         # been created before. This will overwrite the field udf_created_containers.

clarity-ext-scripts/clarity_ext_scripts/covid/partner_api_client.py

@@ -1,4 +1,3 @@
-
 import base64
 from datetime import datetime
 from luhn import verify as mod10verify
@@ -400,6 +399,25 @@ def post_diagnosis_report(self, service_request_id, diagnosis_result, analysis_r
             log.error(e.message)
             raise e
 
+    def get_by_reference(self, ref):
+        """
+        Get's a resource by reference, such as Organization/123 or Patient/345
+
+        :ref: The reference, e.g. Patient/123
+        """
+        url = "{}/{}".format(self._base_url, ref)
+        headers = self._generate_headers()
+        response = self._session.get(url=url, headers=headers)
+        if response.status_code != 200:
+            print(response.text)
+            raise PartnerClientAPIException(
+                "Couldn't get resource '{}', status code: {}".format(
+                    url, response.status_code))
+        return response.json()
+
+    def get_org_uri_by_name(self, name):
+        return ORG_URI_BY_NAME[name]
+
     def _create_payload(self, service_request_id, diagnosis_result, analysis_results):
         # TODO Need to think about if this needs refactoring later, to more easily support multiple
         #      analysis types. This is going to be rather unwieldy to add more as it is implemented

clarity-ext-scripts/clarity_ext_scripts/covid/report_results.py

@@ -6,7 +6,7 @@
     PartnerClientAPIException)
 from clarity_ext_scripts.covid.rtpcr_analysis_service import FAILED_STATES
 from clarity_ext_scripts.covid.controls import Controls
-from clarity_ext_scripts.covid.utils import KNMClient
+from clarity_ext_scripts.covid.services.knm_service import KNMClientFromExtension
 
 
 logger = logging.getLogger(__name__)
@@ -70,7 +70,7 @@ def is_control(self, sample):
         return False
 
     def execute(self):
-        self.client = KNMClient(self) 
+        self.client = KNMClientFromExtension(self)
         for plate in self.context.input_containers:
             for well in plate.occupied:
                 already_uploaded = False

clarity-ext-scripts/clarity_ext_scripts/covid/report_to_sminet.py

@@ -0,0 +1,131 @@
+from datetime import datetime
+import logging
+from sminet_client import SampleMaterial, SmiNetError
+from clarity_ext.extensions import GeneralExtension
+from clarity_ext_scripts.covid.utils import CtmrCovidSubstanceInfo
+from clarity_ext_scripts.covid.services.sminet_service import SmiNetService
+from clarity_ext_scripts.covid.services.knm_service import KNMSampleAccessor
+from clarity_ext_scripts.covid.services.knm_sminet_service import (
+    KNMSmiNetIntegrationService, IntegrationError, UnregisteredPatient)
+from clarity_ext_scripts.covid.partner_api_client import (
+    COVID_RESPONSE_POSITIVE, PartnerClientAPIException)
+
+
+UDF_TRUE = "Yes"
+logger = logging.getLogger(__name__)
+
+
+def should_report(substance):
+    """
+    A substance is ignored from SmiNet reporting if it's a control, has been run before or doesn't
+    have a positive result. It is also ignored if the research engineer has set the UDF
+    "SmiNet status" to "ignore"
+
+    Samples that should be reported must be in the SmiNet state `retry` or `error` or None
+    """
+    if substance.is_control:
+        return False
+
+    if substance.submitted_sample.udf_rtpcr_covid19_result_latest != COVID_RESPONSE_POSITIVE:
+        return False
+
+    if substance.sminet_status in [SmiNetService.STATUS_IGNORE, SmiNetService.STATUS_SUCCESS]:
+        return False
+
+    if substance.sminet_status not in [
+            None,
+            SmiNetService.STATUS_RETRY,
+            SmiNetService.STATUS_ERROR]:
+        raise AssertionError(
+            "Unexpected SmiNet status: {}".format(substance.sminet_status))
+
+    return True
+
+
+class Extension(GeneralExtension):
+    """
+    Reports results to SmiNet.
+
+    For all analytes in the step:
+    * If sminet_status is not set:
+        * Checks if it should be imported.
+            * If it shouldn't, updates it to "ignore"
+            * If it should, tries to send the data
+    * If sminet_status is "error", retries it
+    * If sminet_status is "ignore", ignores it
+
+
+    Test data required:
+    * Positive  => success
+    * Negative  => ignore
+    * Failed    => ignore
+    * Anonymous => ignore
+    """
+
+    def report(self, substance):
+        """
+        Reports this substance to SmiNet and updates the status in the LIMS.
+        """
+
+        org_referral_code = substance.submitted_sample.name.split("_")[0]
+        date_arrival = substance.submitted_sample.api_resource.date_received
+        date_arrival = datetime.strptime(date_arrival, "%Y-%m-%d")
+
+        sample = KNMSampleAccessor(substance.submitted_sample.udf_knm_org_uri,
+                                   org_referral_code,
+                                   date_arrival,
+                                   SampleMaterial.SVALG)
+
+        integration = KNMSmiNetIntegrationService(self.config)
+        lab_result = SmiNetService.create_scov2_positive_lab_result()
+
+        error_msg = ""
+
+        try:
+            integration.export_to_sminet(sample,
+                                         doctor_name="Lars Engstrand",
+                                         lab_result=lab_result,
+                                         sample_free_text="Anamnes: Personalprov")
+            status = SmiNetService.STATUS_SUCCESS
+        except UnregisteredPatient:
+            status = SmiNetService.STATUS_IGNORE
+        except PartnerClientAPIException as knm_error:
+            error_msg = knm_error.message
+            self.usage_error_defer("Error while fetching data from KNM",
+                                   substance.submitted_sample.name)
+            status = SmiNetService.STATUS_ERROR
+        except SmiNetError as sminet_error:
+            error_msg = sminet_error.message
+            self.usage_error_defer("Error while uploading sample to SmiNet",
+                                   substance.submitted_sample.name)
+            status = SmiNetService.STATUS_ERROR
+        except IntegrationError as int_error:
+            self.usage_error_defer("Error while uploading sample to SmiNet",
+                                   substance.submitted_sample.name)
+            error_msg = int_error.message
+            status = SmiNetService.STATUS_ERROR
+
+        timestamp = datetime.now().strftime("%y%m%dT%H%M%S")
+
+        substance.submitted_sample.udf_map.force("SmiNet status", status)
+        substance.substance.udf_map.force("SmiNet status", status)
+
+        substance.submitted_sample.udf_map.force(
+            "SmiNet uploaded date", timestamp)
+        substance.submitted_sample.udf_map.force("SmiNet artifact source",
+                                                 substance.substance.api_resource.uri)
+        substance.submitted_sample.udf_map.force(
+            "SmiNet last error", error_msg)
+        self.context.update(substance.submitted_sample)
+        self.context.update(substance.substance)
+        self.context.commit()
+
+    def execute(self):
+        for plate in self.context.input_containers:
+            for well in plate.occupied:
+                substance = CtmrCovidSubstanceInfo(well.artifact)
+                if should_report(substance):
+                    self.report(substance)
+
+    def integration_tests(self):
+        yield "24-48808"

clarity-ext-scripts/clarity_ext_scripts/covid/services/knm_service.py

@@ -0,0 +1,92 @@
+from clarity_ext_scripts.covid.partner_api_client import PartnerAPIV7Client
+from clarity_ext.utils import lazyprop
+
+
+class NoSupportedCountyCodeFound(Exception):
+    pass
+
+
+class KNMService(object):
+    NoSupportedCountyCodeFound = NoSupportedCountyCodeFound
+
+    def __init__(self, config):
+        self.config = KNMConfig(config)
+        self.client = PartnerAPIV7Client(**self.config)
+
+
+def KNMConfig(config):
+    """
+    Creates config required for KNM from the clarity-ext config (which has more than that)
+    """
+    return {
+        key: config[key]
+        for key in [
+            "test_partner_base_url",
+            "test_partner_code_system_base_url",
+            "test_partner_user",
+            "test_partner_password"
+        ]
+    }
+
+
+def KNMClientFromExtension(extension):
+    # A factory for a KnmClient from an extension
+    config = KNMConfig(extension.config)
+    return PartnerAPIV7Client(**config)
+
+
+class KNMSampleAccessor(object):
+    """
+    Describes a sample/analyte in the LIMS that has originally come via the KNM workflow
+    """
+
+    def __init__(self, org_uri, org_referral_code, date_arrival, material):
+        """
+        :org_uri: The organization URI as defined by KNM
+        :org_referral_code: The referral code within the organization
+        :date_arrival: The date the sample was added to the LIMS
+        :material: The material, one of the constants in SampleMaterialType
+        """
+        self.org_uri = org_uri
+        self.org_referral_code = org_referral_code
+        self.date_arrival = date_arrival
+        self.material = material
+
+    @classmethod
+    def create_from_lims_sample(cls, sample):
+        raise NotImplementedError()
+
+
+class ServiceRequestProvider(object):
+    """
+    Represents a service request. Has methods to retrieve all data we require on it. Gives
+    a higher level abstraction of the api for readability.
+    """
+
+    def __init__(self, client, org_uri, org_referral_code):
+        self.client = client
+        self.org_uri = org_uri
+        self.org_referral_code = org_referral_code
+
+    @lazyprop
+    def service_request(self):
+        # The service_request json response
+        return self.client.search_for_service_request(self.org_uri, self.org_referral_code)
+
+    @lazyprop
+    def patient(self):
+        # The patient data corresponding with the service request
+        return self.client.get_by_reference(self.patient_ref)
+
+    @lazyprop
+    def organization(self):
+        managing_organization = self.patient["managingOrganization"]
+        return self.client.get_by_reference(managing_organization["reference"])
+
+    @property
+    def patient_ref(self):
+        # A reference identifying the patient
+        return self.service_request["resource"]["subject"]["reference"]
+
+    def __str__(self):
+        return "{}|{}".format(self.org_uri, self.org_referral_code)