From 2589d0252a847ec987b0e09e37323f7126952800 Mon Sep 17 00:00:00 2001 From: Steinar Sturlaugsson Date: Wed, 27 May 2020 08:49:16 +0200 Subject: [PATCH 01/14] SmiNet's xsd at the time of client development --- .../covid/sminet/SmiNetLabExport.xsd | 615 ++++++++++++++++++ 1 file changed, 615 insertions(+) create mode 100644 clarity-ext-scripts/clarity_ext_scripts/covid/sminet/SmiNetLabExport.xsd diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/sminet/SmiNetLabExport.xsd b/clarity-ext-scripts/clarity_ext_scripts/covid/sminet/SmiNetLabExport.xsd new file mode 100644 index 0000000..4387dea --- /dev/null +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/sminet/SmiNetLabExport.xsd @@ -0,0 +1,615 @@ + + + + + _SmiNetLabExport XML Schema v1.4_ + + Denna fil fungerar som validering [XML Schema (.xsd)] f�r + xmlfiler som skall skickas till SmiNet. En exportfil som + skall skickas till SmiNet kommer att valideras mot detta + xml-schema innan parsning. Det rekomenderas att validera + filen innan s�ndning f�r b�ttre felhantering. + + Den senaste versionen av denna fil tillg�nglig via: + http://@distribution.ip@/xml-schemas/SmiNetLabExport.xsd. + + SmiNet-koder/specifikationer och mappningar finns i dokumentet + "The latest specifikation code map" som finns att h�mta h�r: + http://sminet.folkhalsomyndigheten.se/LabExporter/index.html + + F�r ytterliggare information, kontakta SMI: + sminet@folkhalsomyndigheten.se + + + + + + + + + + + + SmiNetLabExporters datumformat (����-MM-DD). + + + + + + + + + + + + SmiNetLabExporters datum- och tidformat + (����-MM-DD TT:MM:SS). + + + + + + + + + + + + En str�ng, max 15 tecken l�ng. + + + + + + + + + + + + En str�ng, max 25 tecken l�ng. + + + + + + + + + + + + En str�ng, max 40 tecken l�ng. + + + + + + + + + + + + En str�ng, max 255 tecken l�ng. + + + + + + + + + + + + En str�ng, max 1500 tecken l�ng. + + + + + + + + + + + + DoctorType inneh�ller information om en l�kare, antingen + ansvarig l�kare p� det inskickande laboratoriet eller + behandlande l�kare p� kliniken. + + + + + + + + + + + + + + + + + + + + Rootelementet i xml-dokumentet. Varje exportfil (XML-fil) + m�ste inneh�lla ett och endast ett s�dant element. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avser en anm�lans status. F�rklaring: + 1. Slutsvar. + 2. Komplettering kommer. + 3. Makulering av en tidigare anm�lan. + 4. Komplettering av en tidigare anm�lan. + + + + + + + + + + + + + + Avser unders�kningsmatrial. (Notera att det �r till�tet att + byta ut '�' och '�' mot 'a' samt att '�' kan bytas ut mot 'o'. + Detta alternativ finns om det skulle bli problem med + valideringen pga konstigheter i teckentabeller.) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Inneh�ller information om laboratoriet som skickade provet + f�r referens samt provnumret hos detta lab. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avser smittskyddsl�karens landstingsbokstav. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avser personnummer eller i k�nsliga �renden en rikskod, annat nummer alt. sammordningsnummer. + Fyra olika format godk�nns: + 1) (Personnummer) 12 siffror och 13 tecken ('-' avgr�nsare), exempel: "19671208-1123". + 2) (Rikskod) Exempel: "1967-1123" ('-' avgr�nsare). + 3) (annat nummer) �vriga format (max 20 tecken). + 4) (samordningsnummer) Formatet (YY)YYMMDD-XXXX d�r DD �r ett tal mellan 61-91 och d�r kontrollsiffran st�mmer p� samma vis som f�r personnummer. + + + + + + + + + + + Avser patientens k�n ('o'/'O' = Ok�nt). + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avser diagnostisk metod enligt kriterielista i Epidaktuellt + 8/95. (Notera att flera DiagnosticMethod listade efter varandra + �r till�tna.) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Definierar vilken laboratoriediagnos som avses anm�las. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 6340896d7cbb541ef675684a12924c8c52ebde34 Mon Sep 17 00:00:00 2001 From: Steinar Sturlaugsson Date: Thu, 28 May 2020 13:03:02 +0200 Subject: [PATCH 02/14] SmiNet client A Python package for sending documents to SmiNet --- .github/workflows/pythonapp.yml | 5 + README.md | 4 + .../covid/sminet/SmiNetLabExport.xsd | 615 ------------------ clarity-ext-scripts/requirements.txt | 2 +- clarity-ext-scripts/setup.sh | 1 + .../tests/integration/test_sminet.py | 52 -- .../tests/test_sminet_client.py | 28 - sminet-client/README.md | 8 + sminet-client/setup.py | 23 + .../sminet_client/__init__.py | 327 +++++++--- .../tests/fixtures}/valid_request.xml | 14 +- .../tests/integration/test_sminet_client.py | 96 +++ sminet-client/tests/unit/__init__.py | 0 .../tests/unit/test_sminet_client.py | 38 ++ 14 files changed, 418 insertions(+), 795 deletions(-) delete mode 100644 clarity-ext-scripts/clarity_ext_scripts/covid/sminet/SmiNetLabExport.xsd delete mode 100644 clarity-ext-scripts/tests/integration/test_sminet.py delete mode 100644 clarity-ext-scripts/tests/test_sminet_client.py create mode 100644 sminet-client/README.md create mode 100644 sminet-client/setup.py rename clarity-ext-scripts/clarity_ext_scripts/covid/sminet_client.py => sminet-client/sminet_client/__init__.py (67%) rename {clarity-ext-scripts/tests/fixtures/sminet => sminet-client/tests/fixtures}/valid_request.xml (74%) create mode 100644 sminet-client/tests/integration/test_sminet_client.py create mode 100644 sminet-client/tests/unit/__init__.py create mode 100644 sminet-client/tests/unit/test_sminet_client.py diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index b22ac48..51b0ceb 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -19,6 +19,9 @@ jobs: uses: actions/setup-python@v1 with: python-version: 2.7 + - name: Add dummy genologicsrc + run: | + echo "[genologics]\nBASEURI=https://url\nUSERNAME=username\nPASSWORD=pass\n" > ~/.genologicsrc - name: Install dependencies run: | ./clarity-ext-scripts/setup.sh @@ -26,3 +29,5 @@ jobs: run: | pip install pytest pytest ./clarity-ext-scripts/tests + pytest ./sminet-client/tests + diff --git a/README.md b/README.md index 12912c9..c506f3e 100644 --- a/README.md +++ b/README.md @@ -68,5 +68,9 @@ The [clarity-ext](https://github.com/molmed/clarity-ext) package is designed to Extensions that use this framework are all in ./clarity-ext-scripts. Refer to the [README](./clarity-ext-scripts/README.md) for more information on this package. +## SmiNet + +There is a client for SmiNet integration in ./sminet_client/. Refer to the [README](./sminet_client/README.md) for more information on the package. + ## More information More information about the LIMS system including details on the Miniconda installation and starting the server may be found [here](https://github.com/ctmrbio/wiki/wiki/CTMR-LIMS-(PROD-and-STAGE)). diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/sminet/SmiNetLabExport.xsd b/clarity-ext-scripts/clarity_ext_scripts/covid/sminet/SmiNetLabExport.xsd deleted file mode 100644 index 4387dea..0000000 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/sminet/SmiNetLabExport.xsd +++ /dev/null @@ -1,615 +0,0 @@ - - - - - _SmiNetLabExport XML Schema v1.4_ - - Denna fil fungerar som validering [XML Schema (.xsd)] f�r - xmlfiler som skall skickas till SmiNet. En exportfil som - skall skickas till SmiNet kommer att valideras mot detta - xml-schema innan parsning. Det rekomenderas att validera - filen innan s�ndning f�r b�ttre felhantering. - - Den senaste versionen av denna fil tillg�nglig via: - http://@distribution.ip@/xml-schemas/SmiNetLabExport.xsd. - - SmiNet-koder/specifikationer och mappningar finns i dokumentet - "The latest specifikation code map" som finns att h�mta h�r: - http://sminet.folkhalsomyndigheten.se/LabExporter/index.html - - F�r ytterliggare information, kontakta SMI: - sminet@folkhalsomyndigheten.se - - - - - - - - - - - - SmiNetLabExporters datumformat (����-MM-DD). - - - - - - - - - - - - SmiNetLabExporters datum- och tidformat - (����-MM-DD TT:MM:SS). - - - - - - - - - - - - En str�ng, max 15 tecken l�ng. - - - - - - - - - - - - En str�ng, max 25 tecken l�ng. - - - - - - - - - - - - En str�ng, max 40 tecken l�ng. - - - - - - - - - - - - En str�ng, max 255 tecken l�ng. - - - - - - - - - - - - En str�ng, max 1500 tecken l�ng. - - - - - - - - - - - - DoctorType inneh�ller information om en l�kare, antingen - ansvarig l�kare p� det inskickande laboratoriet eller - behandlande l�kare p� kliniken. - - - - - - - - - - - - - - - - - - - - Rootelementet i xml-dokumentet. Varje exportfil (XML-fil) - m�ste inneh�lla ett och endast ett s�dant element. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Avser en anm�lans status. F�rklaring: - 1. Slutsvar. - 2. Komplettering kommer. - 3. Makulering av en tidigare anm�lan. - 4. Komplettering av en tidigare anm�lan. - - - - - - - - - - - - - - Avser unders�kningsmatrial. (Notera att det �r till�tet att - byta ut '�' och '�' mot 'a' samt att '�' kan bytas ut mot 'o'. - Detta alternativ finns om det skulle bli problem med - valideringen pga konstigheter i teckentabeller.) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Inneh�ller information om laboratoriet som skickade provet - f�r referens samt provnumret hos detta lab. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Avser smittskyddsl�karens landstingsbokstav. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Avser personnummer eller i k�nsliga �renden en rikskod, annat nummer alt. sammordningsnummer. - Fyra olika format godk�nns: - 1) (Personnummer) 12 siffror och 13 tecken ('-' avgr�nsare), exempel: "19671208-1123". - 2) (Rikskod) Exempel: "1967-1123" ('-' avgr�nsare). - 3) (annat nummer) �vriga format (max 20 tecken). - 4) (samordningsnummer) Formatet (YY)YYMMDD-XXXX d�r DD �r ett tal mellan 61-91 och d�r kontrollsiffran st�mmer p� samma vis som f�r personnummer. - - - - - - - - - - - Avser patientens k�n ('o'/'O' = Ok�nt). - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Avser diagnostisk metod enligt kriterielista i Epidaktuellt - 8/95. (Notera att flera DiagnosticMethod listade efter varandra - �r till�tna.) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Definierar vilken laboratoriediagnos som avses anm�las. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/clarity-ext-scripts/requirements.txt b/clarity-ext-scripts/requirements.txt index 78c3b64..eb3e87d 100644 --- a/clarity-ext-scripts/requirements.txt +++ b/clarity-ext-scripts/requirements.txt @@ -3,4 +3,4 @@ mock==3.0.5 pytest==4.6.9 requests[socks]==2.23.0 retry==0.9.2 -xlwt==1.3.0 +suds==0.4 diff --git a/clarity-ext-scripts/setup.sh b/clarity-ext-scripts/setup.sh index 101da66..e562190 100755 --- a/clarity-ext-scripts/setup.sh +++ b/clarity-ext-scripts/setup.sh @@ -11,3 +11,4 @@ pip install -e ./repos/clarity-ext pip install -r $SCRIPT_DIR/requirements.txt pip install -e $SCRIPT_DIR/. +pip install -e $SCRIPT_DIR/../sminet-client diff --git a/clarity-ext-scripts/tests/integration/test_sminet.py b/clarity-ext-scripts/tests/integration/test_sminet.py deleted file mode 100644 index 7a7cd09..0000000 --- a/clarity-ext-scripts/tests/integration/test_sminet.py +++ /dev/null @@ -1,52 +0,0 @@ -import os -from uuid import uuid4 -import random -import string -import yaml -import pytest -from datetime import datetime -from clarity_ext_scripts.covid.sminet_client import (create_covid_request, SampleInfo, - ReferringClinic, Patient, Doctor, - SmiNetClient) - - -def not_setup_on_this_server(): - pytest.xfail( - "This test requires sminet to be setup in your clarity-ext config") - - -def get_sminet_client(): - path = os.path.expanduser("~/.config/clarity-ext/clarity-ext.config") - if not os.path.exists(path): - not_setup_on_this_server() - - with open(path) as f: - client_config = yaml.safe_load(f) - - if not "sminet_url" in client_config: - not_setup_on_this_server() - return SmiNetClient.SmiNetClientFromConfig(client_config) - - -def test_can_create_request(): - """ - NOTE: We work against a whitelist on a jump server. Information available from team lead. - NOTE: Reads config from your clarity-ext config which should point to the test environment. - If sminet config is not available, the test is ignored. This is because the test - can't run on the build server because it's not whitelisted at the moment. - """ - - client = get_sminet_client() - constant_date = datetime(2020, 4, 30) - request_created = datetime(2020, 5, 18, 18, 11, 6) - prefix = "int-tests-" - rnd = "".join(random.choice(string.ascii_uppercase + string.digits) - for _ in range(25 - len(prefix))) - sample_id = prefix + rnd - sample_info = SampleInfo(status=1, sample_id=sample_id, sample_date_arrival=constant_date, - sample_date_referral=constant_date, sample_material="Svalg", - sample_free_text_referral="Anamnes: Personalprov") - clinic = ReferringClinic("Clinic name", "", "C", Doctor("Some doctor")) - patient = Patient("1234", "k", "Some Name", 23) - reporting_doctor = Doctor("Lars Engstrand") - client.create(sample_info, clinic, patient, reporting_doctor) diff --git a/clarity-ext-scripts/tests/test_sminet_client.py b/clarity-ext-scripts/tests/test_sminet_client.py deleted file mode 100644 index 076c148..0000000 --- a/clarity-ext-scripts/tests/test_sminet_client.py +++ /dev/null @@ -1,28 +0,0 @@ -from datetime import datetime -from clarity_ext_scripts.covid.sminet_client import (create_covid_request, SampleInfo, - ReferringClinic, Patient, Doctor) -import lxml.etree as ET -import os - - -def test_can_create_expected_request(): - constant_date = datetime(2020, 4, 30) - request_created = datetime(2020, 5, 18, 18, 11, 6) - - sample_info = SampleInfo(status=1, sample_id="123", sample_date_arrival=constant_date, - sample_date_referral=constant_date, sample_material="Svalg", - sample_free_text_referral="Anamnes: Personalprov") - clinic = ReferringClinic("Clinic name", "", "C", Doctor("Some doctor")) - patient = Patient("1234", "k", "Some Name", 23) - reporting_doctor = Doctor("Lars Engstrand") - request = create_covid_request( - sample_info, clinic, patient, reporting_doctor, request_created) - - fixtures = os.path.join(os.path.dirname(__file__), "fixtures") - - # For debug reasons one can uncomment this: - # with open(os.path.join(fixtures, "sminet", "generated.xml"), "w") as fs: - # fs.write(request) - - with open(os.path.join(fixtures, "sminet", "valid_request.xml")) as fs: - assert fs.read() == request diff --git a/sminet-client/README.md b/sminet-client/README.md new file mode 100644 index 0000000..478d752 --- /dev/null +++ b/sminet-client/README.md @@ -0,0 +1,8 @@ +# SmiNet client + +This package contains a client for SmiNet, Public Health Agency of Sweden's web service for reporting laboratory results. + +## Installation + +pip install -e . + diff --git a/sminet-client/setup.py b/sminet-client/setup.py new file mode 100644 index 0000000..edfd6f5 --- /dev/null +++ b/sminet-client/setup.py @@ -0,0 +1,23 @@ +from setuptools import find_packages, setup + +setup( + name='sminet-client', + version='1.0.0', + packages=find_packages(exclude=['tests']), + requirements=['suds'], + zip_safe=False, + platforms='any', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', + 'Operating System :: POSIX', + 'Operating System :: MacOS', + 'Operating System :: Unix', + 'Operating System :: Windows', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Topic :: Software Development :: Libraries :: Python Modules', + ] +) diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/sminet_client.py b/sminet-client/sminet_client/__init__.py similarity index 67% rename from clarity-ext-scripts/clarity_ext_scripts/covid/sminet_client.py rename to sminet-client/sminet_client/__init__.py index 9bffb2d..46ee453 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/sminet_client.py +++ b/sminet-client/sminet_client/__init__.py @@ -1,15 +1,23 @@ # -*- coding: utf-8 -*- +import os +import yaml import datetime import requests import logging +import codecs +import base64 +import lxml +from lxml import objectify import lxml.etree as ET +from suds.client import Client +from suds.xsd.doctor import ImportDoctor, Import logger = logging.getLogger(__name__) """ -A Python client for the SmiNet site +A client for the SmiNet site NOTE: Some of the documentation is in Swedish. In those cases, it has been copied directly from @@ -27,7 +35,7 @@ def add_child(parent, elementName, value, validator=None): child_element = ET.SubElement(parent, elementName) if validator: value = validator(value) - child_element.text = str(value) # Must always be cast to string + child_element.text = unicode(value) # Must always be cast to string #### @@ -50,11 +58,19 @@ def IdType(python_string): return python_string +class Gender(object): + MALE = "m" + FEMALE = "k" + UNKNOWN = "o" + + ALL = [MALE, FEMALE, UNKNOWN] + + def SexType(python_string): """ Avser patientens kön ('o'/'O' = Okänt). """ - if python_string.lower() not in ["m", "k", "o"]: + if python_string.lower() not in Gender.ALL: raise SmiNetValidationError( "Unknown sex type {}".format(python_string)) return python_string @@ -71,13 +87,64 @@ def DiagnosticMethod(python_string): return python_string +COUNTY_STOCKHOLM = "Stockholm" +COUNTY_UPPSALA = "Uppsala" +COUNTY_SODERMANLAND = "Sodermanland" +COUNTY_OSTERGOTLAND = "Ostergotland" +COUNTY_JONKOPING = "Jonkoping" +COUNTY_KRONOBERG = "Kronoberg" +COUNTY_KALMAR = "Kalmar" +COUNTY_GOTLAND = "Gotland" +COUNTY_BLEKINGE = "Blekinge" +COUNTY_SKANE = "Skane" +COUNTY_HALLAND = "Halland" +COUNTY_VASTRAGOTALAND = "Vastragotaland" +COUNTY_VARMLAND = "Varmland" +COUNTY_OREBRO = "Orebro" +COUNTY_VASTMANLAND = "Vastmanland" +COUNTY_DALARNA = "Dalarna" +COUNTY_GAVLEBORG = "Gavleborg" +COUNTY_VASTERNORRLAND = "Vasternorrland" +COUNTY_JAMTLAND = "Jamtland" +COUNTY_VASTERBOTTEN = "Vasterbotten" +COUNTY_NORRBOTTEN = "Norrbotten" + + +MAP_COUNTY_CODE_TO_NAME = { + "AB": COUNTY_STOCKHOLM, + "C": COUNTY_UPPSALA, + "D": COUNTY_SODERMANLAND, + "E": COUNTY_OSTERGOTLAND, + "F": COUNTY_JONKOPING, + "G": COUNTY_KRONOBERG, + "H": COUNTY_KALMAR, + "I": COUNTY_GOTLAND, + "K": COUNTY_BLEKINGE, + "M": COUNTY_SKANE, + "N": COUNTY_HALLAND, + "O": COUNTY_VASTRAGOTALAND, + "S": COUNTY_VARMLAND, + "T": COUNTY_OREBRO, + "U": COUNTY_VASTMANLAND, + "W": COUNTY_DALARNA, + "X": COUNTY_GAVLEBORG, + "Y": COUNTY_VASTERNORRLAND, + "Z": COUNTY_JAMTLAND, + "AC": COUNTY_VASTERBOTTEN, + "BD": COUNTY_NORRBOTTEN, + + "OB": COUNTY_VASTRAGOTALAND, + "OG": COUNTY_VASTRAGOTALAND, + "OS": COUNTY_VASTRAGOTALAND, + "OT": COUNTY_VASTRAGOTALAND, +} + + def CountyType(python_string): """ Avser smittskyddsläkarens landstingsbokstav. """ - if python_string not in ["AB", "C", "D", "E", "F", "G", "H", "I", "K", "M", - "N", "O", "OB", "OG", "OS", "OT", "S", "T", "U", "W", - "X", "Y", "Z", "AC", "BD"]: + if python_string not in MAP_COUNTY_CODE_TO_NAME.keys(): raise SmiNetValidationError( "Unknown county code {}".format(python_string)) return python_string @@ -171,19 +238,6 @@ def Version(version_number): return element -def Laboratory(lab_number, lab_name): - """ - :lab_number: Non-negative integer. Each lab that connects with "automatisk överföring" against - SmiNet gets a unique identification number - :lab_name: Name of the laboratory exporting the file. Max 255 characters. - """ - laboratory = ET.Element("laboratory") - add_child(laboratory, "labNumber", lab_number, NonNegativeInteger) - add_child(laboratory, 'labName', lab_name, LongLimitedString) - - return laboratory - - # NOTE: These are our translations of the status in the xsd StatusType documentation, which is in # Swedish. Might be inaccurate. STATUS_FINAL_RESPONSE = 1 @@ -236,6 +290,23 @@ def __str__(self): return ET.tostring(self.to_element(), pretty_print=True) +class Laboratory(SmiNetComplexType): + def __init__(self, lab_number, lab_name): + """ + :lab_number: Non-negative integer. Each lab that connects with "automatisk överföring" against + SmiNet gets a unique identification number + :lab_name: Name of the laboratory exporting the file. Max 255 characters. + """ + self.lab_number = NonNegativeInteger(lab_number) + self.lab_name = LongLimitedString(lab_name) + + def to_element(self, element_name="laboratory"): + laboratory = ET.Element(element_name) + add_child(laboratory, "labNumber", self.lab_number) + add_child(laboratory, 'labName', self.lab_name) + return laboratory + + class ReferringClinic(SmiNetComplexType): def __init__(self, referring_clinic_name, referring_clinic_address, referring_clinic_county, referring_doctor=None): @@ -332,7 +403,6 @@ def to_element(self, element_name="sampleInfo"): class NotificationType(SmiNetComplexType): - def __init__(self, sample_info, reporting_doctor, referring_clinic, patient, lab_result): """ :sample_info: Översiktlig information om provet. @@ -438,7 +508,7 @@ def to_element(self, element_name="patient"): return element -def SmiNetLabExport(created, lab_number, notification): +class SmiNetLabExport(): """ Creates a validated lab export as XML that is acceptable for the SmiNet endpoint. @@ -447,47 +517,38 @@ def SmiNetLabExport(created, lab_number, notification): Original documentation in xsd: Rootelementet i xml-dokumentet. Varje exportfil (XML-fil) måste innehålla ett och endast ett sådant element. """ - element = ET.Element('smiNetLabExport') # TODO: xlmns and xsi - # # smiNetLabExport/version/version-number - element.append(Version("4.0.0")) + def __init__(self, created, laboratory, notification): + self.created = created + self.laboratory = laboratory + self.notification = notification + self.version = Version("4.0.0") - # # smiNetLabExport/dateTimeCreated - date_time_created = ET.Element("dateTimeCreated") - date_time_created.text = SmiNetDateTime(created) - element.append(date_time_created) + def to_element(self, xsd_url): + element = ET.Element('smiNetLabExport') - # # smiNetLabExport/laboratory - element.append(Laboratory(lab_number, "National Pandemic Center at KI")) + # element.attrib['xmlns:xsi'] = "" + element.attrib["{http://www.w3.org/2001/XMLSchema-instance}noNamespaceSchemaLocation"] = xsd_url - element.append(notification.to_element()) - return element + # # smiNetLabExport/version/version-number + element.append(self.version) + # # smiNetLabExport/dateTimeCreated + date_time_created = ET.Element("dateTimeCreated") + date_time_created.text = SmiNetDateTime(self.created) + element.append(date_time_created) -def create_scov2_positive_lab_result(): - lab_diagnosis = LabDiagnosisType("SCOV2", "SARS-CoV-2 (covid-19)") - return LabResult("C", lab_diagnosis) - - -def create_covid_request(sample_info, referring_clinic, patient, reporting_doctor, created=None): - """ - Creates a request to SmiNet that's valid for the Covid project. - - :sample_info: An object created with the SampleInfo factory - :referring_clinic: An object created with the ReferringClinic factory - :patient: An object created with the Patient factory - """ - - if not created: - created = datetime.datetime.now() - - notification = NotificationType(sample_info, reporting_doctor, referring_clinic, patient, - create_scov2_positive_lab_result()) - - lab_number = 91 # TODO, ensure that this is correct - contract = SmiNetLabExport(created, lab_number, notification) + # # smiNetLabExport/laboratory + element.append(self.laboratory.to_element()) + element.append(self.notification.to_element()) + return element - return ET.tostring(contract, pretty_print=True, encoding="ISO-8859-1") + def to_document(self, xsd_url): + """ + Creates an XML string representation that should be accepted by SmiNet + """ + export = self.to_element(xsd_url) + return ET.tostring(export, pretty_print=True, encoding="ISO-8859-1") class SmiNetValidationError(Exception): @@ -499,54 +560,136 @@ class SmiNetRequestError(Exception): class SmiNetClient(object): - def __init__(self, sminet_url, sminet_username, sminet_password, sminet_proxy=None): + + SMINET_ENVIRONMENT_STAGE = "stage" + SMINET_ENVIRONMENT_PROD = "prod" + + def __init__(self, config): """ - :sminet_url: The url - :sminet_username: The username - :sminet_password: The password - :proxy: Proxy info if the client requires socks (we access SmiNet via a jump host in dev) + :config: A configuration object of type SmiNetConfig """ - self.url = sminet_url - if not "https://" in self.url: - raise AssertionError("The SmiNet url should use https") - self.username = sminet_username - self.password = sminet_password - self.proxy = sminet_proxy - def create(self, sample_info, referring_clinic, patient, reporting_doctor, created=None): + self.config = config + self._soap_client = None + + if config.environment == self.SMINET_ENVIRONMENT_STAGE: + self.url = "https://stage.sminet.se/sminetlabxmlzone/services/SmiNetLabXmlZone" + self.xsd_url = "http://stage.sminet.se/xml-schemas/SmiNetLabExport.xsd" + self.wsdl_url = "https://stage.sminet.se/sminetlabxmlzone/services/SmiNetLabXmlZone?wsdl" + elif config.environment == self.SMINET_ENVIRONMENT_PROD: + self.url = "https://sminet.se/sminetlabxmlzone/services/SmiNetLabXmlZone" + self.xsd_url = "http://sminet.se/xml-schemas/SmiNetLabExport.xsd" + self.wsdl_url = "https://sminet.se/sminetlabxmlzone/services/SmiNetLabXmlZone?wsdl" + + @property + def soap_client(self): + if self._soap_client is None: + imp = Import('http://schemas.xmlsoap.org/soap/encoding/') + doctor = ImportDoctor(imp) + self._soap_client = Client(self.wsdl_url, + doctor=doctor) + return self._soap_client + + def is_supported_county_code(self, county_code): + return county_code in MAP_COUNTY_CODE_TO_NAME.keys() + + def create(self, sminet_lab_export, file_name): """ Creates the entry in SmiNet - :sample_info: A SampleInfo instance - :referring_clinic: A ReferringClinic instance - :patient: A Patient instance - :reporting_doctor: A Doctor instance - :created: The date when the request is created. Defaults to now() + :export: An instance of SmiNetLabExport + :file_name: Name of the file in SmiNet's database """ logger.info("Creating a request at SmiNet") headers = {"Content-Type": "application/xml"} - data = create_covid_request( - sample_info, referring_clinic, patient, reporting_doctor, created) - response = requests.post(self.url, data=data, proxies=dict( - https=self.proxy), headers=headers) - if response.status_code != requests.codes.created: - raise SmiNetRequestError(response.text) + doc = sminet_lab_export.to_document(self.xsd_url) + self._send_file(doc, file_name) + + def _parse_xml(self, xml): + """ + Returns a Python object from an XML response string" + """ + xml = codecs.encode(xml, "utf-8") + obj = objectify.fromstring(xml) + return obj + + def _send_file(self, xml_file, file_name): + """ + Sends an XML document to SmiNet. - logger.info("Request created at SmiNet. Response code was={}".format( - response.status_code)) + Raises a SmiNetRequestError if the return code is not zero. - @classmethod - def SmiNetClientFromConfig(cls, extension_config): + :xml_file: A valid xml document as a string. Use SmiNetLabExport to generate + a valid document. + :file_name: The name of the file in SmiNet + """ + base64encoded = base64.b64encode(xml_file) + + resp = self.soap_client.service.submitFile( + self.config.username, self.config.password, file_name, base64encoded) + resp = self._parse_xml(resp) + + if resp.returnCode != 0: + raise SmiNetRequestError(resp.message) + + def validate(self, xml): + """ + Validates an XML contract against the xsd SmiNet provides + """ + xml_validator = lxml.etree.XMLSchema( + file="http://stage.sminet.se/xml-schemas/SmiNetLabExport.xsd") + is_valid = xml_validator.validate(xml) + + if not is_valid: + raise SmiNetValidationError( + "Not able to validate xml against the xsd file") + + +class SmiNetConfig(object): + DEFAULT_PATH_ETC = "/etc/sminet_client/sminet_client.config" + DEFAULT_PATH_USER = "~/.config/sminet_client/sminet_client.config" + + def __init__(self, sminet_username, + sminet_password, + sminet_proxy, + sminet_environment, + sminet_lab_name, + sminet_lab_number, + **kwargs): """ - Creates a SmiNetClient from a clarity-ext extension config (self.config) + Creates a configuration file that's valid for this SmiNet client. + + Searches the search paths for a valid configuration file + + :sminet_username: The username + :sminet_password: The password + :sminet_environment: The environment, e.g. SmiNetClient.SMINET_ENVIRONMENT_STAGE + :proxy: Proxy info if the service is only accessible via a proxy + :lab_name: The name of your lab + :lab_number: The name of your lab """ - filtered = { - key: extension_config[key] - for key in [ - "sminet_url", - "sminet_username", - "sminet_password", - "sminet_proxy"] if key in extension_config - } - return cls(**filtered) + self.username = sminet_username + self.password = sminet_password + self.proxy = sminet_proxy + self.environment = sminet_environment + self.lab_name = sminet_lab_name + self.lab_number = sminet_lab_number + + @classmethod + def create_from_search_paths(cls, paths=None): + if not paths: + paths = [cls.DEFAULT_PATH_ETC, cls.DEFAULT_PATH_USER] + + for path in paths: + path = os.path.expanduser(path) + if os.path.exists(path): + with open(path, "r") as f: + config = yaml.safe_load(f) + return cls(**config) + raise SmiNetConfigNotFoundError( + "No config file found in {}".format(paths)) + + +class SmiNetConfigNotFoundError(Exception): + pass diff --git a/clarity-ext-scripts/tests/fixtures/sminet/valid_request.xml b/sminet-client/tests/fixtures/valid_request.xml similarity index 74% rename from clarity-ext-scripts/tests/fixtures/sminet/valid_request.xml rename to sminet-client/tests/fixtures/valid_request.xml index 7709741..762f07f 100644 --- a/clarity-ext-scripts/tests/fixtures/sminet/valid_request.xml +++ b/sminet-client/tests/fixtures/valid_request.xml @@ -1,12 +1,12 @@ - + 4.0.0 2020-05-18 18:11:06 - 91 - National Pandemic Center at KI + 1 + Some lab @@ -15,23 +15,23 @@ 2020-04-30 2020-04-30 Svalg - Anamnes: Personalprov + Free text - Lars Engstrand + Reporting Doctor Clinic name C - Some doctor + Referring doctor 1234 k - Some Name + Patient Name 23 diff --git a/sminet-client/tests/integration/test_sminet_client.py b/sminet-client/tests/integration/test_sminet_client.py new file mode 100644 index 0000000..6c89dce --- /dev/null +++ b/sminet-client/tests/integration/test_sminet_client.py @@ -0,0 +1,96 @@ +import os +import string +import random +from datetime import datetime +import lxml.etree as ET +import pytest +from sminet_client import (SampleInfo, ReferringClinic, Patient, Doctor, SmiNetConfig, NotificationType, + LabDiagnosisType, LabResult, Laboratory, SmiNetLabExport, ConfigNotFoundError, + SmiNetClient, SmiNetValidationError) + + +""" +Tests an integration to the SmiNet. Requires a configuration file (see README.md) +""" + +try: + config = SmiNetConfig.create_from_search_paths() +except ConfigNotFoundError: + pytest.xfail("This test requires a sminet_client configuration file") + + +def generate_valid_contract_with_random_sample_id(): + """ + Returns a tuple of the parameters required to upload sample info via client.create + """ + + constant_date = datetime(2020, 4, 30) + request_created = datetime(2020, 5, 18, 18, 11, 6) + prefix = "int-tests-" + rnd = "".join(random.choice(string.ascii_uppercase + string.digits) + for _ in range(25 - len(prefix))) + sample_id = prefix + rnd + + sample_info = SampleInfo(status=1, sample_id=sample_id, sample_date_arrival=constant_date, + sample_date_referral=constant_date, sample_material="Svalg", + sample_free_text_referral="Extra info") + referring_clinic = ReferringClinic("Clinic name", "", "C", Doctor("Some doctor")) + patient = Patient("121212-1212", "k", "Tolvan Tolvansson", 23) + reporting_doctor = Doctor("Reporting Doctor") + + lab_diagnosis = LabDiagnosisType("SCOV2", "SARS-CoV-2 (covid-19)") + lab_result = LabResult("C", lab_diagnosis) + notification = NotificationType(sample_info, reporting_doctor, referring_clinic, patient, + lab_result) + laboratory = Laboratory(config.lab_number, config.lab_name) + export = SmiNetLabExport(request_created, laboratory, notification) + return export + + +def test_can_create_request(): + """ + Tests creating a request. The test is xfailed if the configuration is not available + """ + client = SmiNetClient(config) + export = generate_valid_contract_with_random_sample_id() + timestamp = datetime.now().strftime("%y%m%dT%H%M%S") + file_name = "{}-{}".format(timestamp, export.notification.sample_info.sample_id) + client.create(export, file_name) + + +def test_can_create_request_with_missing_info(): + """ + Ensure that we can create info were the following data is missing + """ + client = SmiNetClient(config) + export = generate_valid_contract_with_random_sample_id() + + export.notification.patient.patient_age = "1" # This can not be empty + export.notification.patient.patient_age = None + export.notification.patient.patient_name = "" + + timestamp = datetime.now().strftime("%y%m%dT%H%M%S") + file_name = "some-empty-{}-{}".format(timestamp, export.notification.sample_info.sample_id) + client.create(export, file_name) + + +def test_covid_request_xml_is_valid_against_xsd_schema(): + client = SmiNetClient(config) + export = generate_valid_contract_with_random_sample_id() + xml = export.to_element(client.SMINET_ENVIRONMENT_STAGE) + client.validate(xml) + + +def test_covid_request_xml_is_not_valid_against_xsd_schema(): + """ + Ensure that we get a validation error if making a significant change to the xml doc + """ + client = SmiNetClient(config) + export = generate_valid_contract_with_random_sample_id() + xml = export.to_element(client.SMINET_ENVIRONMENT_STAGE) + element = ET.Element("extra") + xml.append(element) + with pytest.raises(SmiNetValidationError): + client.validate(xml) + + diff --git a/sminet-client/tests/unit/__init__.py b/sminet-client/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sminet-client/tests/unit/test_sminet_client.py b/sminet-client/tests/unit/test_sminet_client.py new file mode 100644 index 0000000..76cb53e --- /dev/null +++ b/sminet-client/tests/unit/test_sminet_client.py @@ -0,0 +1,38 @@ +import os +from lxml import etree +from datetime import datetime +from sminet_client import (SampleInfo, ReferringClinic, Patient, Doctor, SmiNetConfig, NotificationType, + LabDiagnosisType, LabResult, Laboratory, SmiNetLabExport) + + +def get_fixture(fname): + fixtures = os.path.join(os.path.dirname(__file__), "..", "fixtures") + with open(os.path.join(fixtures, fname)) as fs: + return fs.read() + + +def create_document(): + constant_date = datetime(2020, 4, 30) + request_created = datetime(2020, 5, 18, 18, 11, 6) + + sample_info = SampleInfo(status=1, sample_id="123", sample_date_arrival=constant_date, + sample_date_referral=constant_date, sample_material="Svalg", + sample_free_text_referral="Free text") + referring_clinic = ReferringClinic("Clinic name", "", "C", Doctor("Referring doctor")) + patient = Patient("1234", "k", "Patient Name", 23) + reporting_doctor = Doctor("Reporting Doctor") + + lab_diagnosis = LabDiagnosisType("SCOV2", "SARS-CoV-2 (covid-19)") + lab_result = LabResult("C", lab_diagnosis) + notification = NotificationType(sample_info, reporting_doctor, referring_clinic, patient, + lab_result) + laboratory = Laboratory(1, "Some lab") + export = SmiNetLabExport(request_created, laboratory, notification) + return export.to_document("http://stage.sminet.se/xml-schemas/SmiNetLabExport.xsd") + +def test_can_create_expected_request(): + expected = get_fixture("valid_request.xml") + actual = create_document() + assert expected == actual + + From 88abaeedfe1f7f4fdcbdfad384f0867353f85379 Mon Sep 17 00:00:00 2001 From: Steinar Sturlaugsson Date: Fri, 29 May 2020 13:56:34 +0200 Subject: [PATCH 03/14] Refactor KNMClient --- .../assign_unregistered_to_anonymous.py | 8 ++++---- .../covid/discard_samples.py | 6 ++++-- .../covid/import_samples.py | 4 ++-- .../covid/report_results.py | 4 ++-- .../clarity_ext_scripts/covid/utils.py | 18 ++---------------- .../covid/validate_discarded_samples.py | 6 +++--- .../covid/validate_sample_creation_list.py | 6 +++--- 7 files changed, 20 insertions(+), 32 deletions(-) diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/create_samples/assign_unregistered_to_anonymous.py b/clarity-ext-scripts/clarity_ext_scripts/covid/create_samples/assign_unregistered_to_anonymous.py index f811719..2091b85 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/create_samples/assign_unregistered_to_anonymous.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/create_samples/assign_unregistered_to_anonymous.py @@ -1,9 +1,9 @@ from uuid import uuid4 from clarity_ext.extensions import GeneralExtension from clarity_ext_scripts.covid.create_samples.common import ValidatedSampleListFile -from clarity_ext_scripts.covid.utils import KNMClient +from clarity_ext_scripts.covid.knm_service import KNMClientFromExtension from clarity_ext_scripts.covid.partner_api_client import ( - ORG_URI_BY_NAME, TESTING_ORG, CouldNotCreateServiceRequest, ServiceRequestAlreadyExists) + ORG_URI_BY_NAME, TESTING_ORG, CouldNotCreateServiceRequest, ServiceRequestAlreadyExists) class Extension(GeneralExtension): @@ -11,7 +11,7 @@ class Extension(GeneralExtension): Goes through the 'Validated sample list' ensuring that each sample that has the unregistered status gets an anonymous service request id. - Uploads a new file to the validated sample list file handle. The name of it will be on the form + Uploads a new file to the validated sample list file handle. The name of it will be on the form 'validated_sample_list_no_unregistered_.csv' Note that the user is not required to run this extension. They can create samples directly @@ -20,7 +20,7 @@ class Extension(GeneralExtension): """ def execute(self): - client = KNMClient(self) + client = KNMClientFromExtension(self) validated_sample_list = ValidatedSampleListFile.create_from_context( self.context) no_unregistered = ValidatedSampleListFile(validated_sample_list.csv) diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/discard_samples.py b/clarity-ext-scripts/clarity_ext_scripts/covid/discard_samples.py index 36c21a0..de52b2a 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/discard_samples.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/discard_samples.py @@ -6,7 +6,7 @@ PartnerClientAPIException) from clarity_ext_scripts.covid.rtpcr_analysis_service import FAILED_STATES from clarity_ext_scripts.covid.controls import Controls -from clarity_ext_scripts.covid.utils import CtmrCovidSubstanceInfo, KNMClient +from clarity_ext_scripts.covid.utils import CtmrCovidSubstanceInfo, KNMClientFromExtension from clarity_ext_scripts.covid.import_samples import BaseCreateSamplesExtension logger = logging.getLogger(__name__) @@ -61,18 +61,20 @@ def report(self, analyte): self.context.commit() def execute(self): - self.client = KNMClient(self) + self.client = KNMClientFromExtension(self) for plate in self.context.input_containers: for well in plate.occupied: already_uploaded = False try: already_uploaded = well.artifact.udf_knm_result_uploaded == UDF_TRUE + except AttributeError: pass if already_uploaded: logger.info("Analyte {} has already been uploaded".format( well.artifact.name)) + continue self.report(well.artifact) diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/import_samples.py b/clarity-ext-scripts/clarity_ext_scripts/covid/import_samples.py index 00b73fc..77e5f72 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/import_samples.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/import_samples.py @@ -1,6 +1,6 @@ from clarity_ext.domain import Container, Sample from clarity_ext_scripts.covid.controls import controls_barcode_generator, Controls -from clarity_ext_scripts.covid.utils import KNMClient +from clarity_ext_scripts.covid.knm_service import KNMClientFromExtension from clarity_ext_scripts.covid.create_samples.common import BaseCreateSamplesExtension @@ -118,7 +118,7 @@ def execute(self): """ Creates samples from a validated import file """ - self.client = KNMClient(self) + self.client = KNMClientFromExtension(self) # This is for debug reasons only. Set this to True to create samples even if they have # been created before. This will overwrite the field udf_created_containers. diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/report_results.py b/clarity-ext-scripts/clarity_ext_scripts/covid/report_results.py index 9eaadf1..20ee968 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/report_results.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/report_results.py @@ -6,7 +6,7 @@ PartnerClientAPIException) from clarity_ext_scripts.covid.rtpcr_analysis_service import FAILED_STATES from clarity_ext_scripts.covid.controls import Controls -from clarity_ext_scripts.covid.utils import KNMClient +from clarity_ext_scripts.covid.knm_service import KNMClientFromExtension logger = logging.getLogger(__name__) @@ -70,7 +70,7 @@ def is_control(self, sample): return False def execute(self): - self.client = KNMClient(self) + self.client = KNMClientFromExtension(self) for plate in self.context.input_containers: for well in plate.occupied: already_uploaded = False diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/utils.py b/clarity-ext-scripts/clarity_ext_scripts/covid/utils.py index b355bc4..56b31a8 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/utils.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/utils.py @@ -85,7 +85,6 @@ class CtmrCovidSubstanceInfo(object): STATUS_DISCARD = "DISCARD" STATUS_DISCARDED_AND_REPORTED = "DISCARDED_AND_REPORTED" - def __init__(self, substance): """ @@ -126,24 +125,11 @@ def control_type(self): if isinstance(self.substance, Sample): return self._deduce_control_type_from_sample(self.substance) elif isinstance(self.substance, Analyte): - control_type = self._deduce_control_type_from_analyte_name(self.substance.name) + control_type = self._deduce_control_type_from_analyte_name( + self.substance.name) if control_type: return control_type return self._deduce_control_type_from_sample(self.substance.sample()) else: raise NotImplementedError("Not implemented substance type {}".format( type(self.substance))) - - - -def KNMClient(extension): - # A factory for a KnmClient from an extension - config = { - key: extension.config[key] - for key in [ - "test_partner_base_url", "test_partner_code_system_base_url", - "test_partner_user", "test_partner_password" - ] - } - return PartnerAPIV7Client(**config) - diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/validate_discarded_samples.py b/clarity-ext-scripts/clarity_ext_scripts/covid/validate_discarded_samples.py index f954e7e..2fc294f 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/validate_discarded_samples.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/validate_discarded_samples.py @@ -2,7 +2,7 @@ from clarity_ext_scripts.covid.create_samples.common import ( BaseValidateRawSampleListExtension, BaseRawSampleListFile, BUTTON_TEXT_ASSIGN_UNREGISTERED_TO_ANONYMOUS) -from clarity_ext_scripts.covid.utils import KNMClient +from clarity_ext_scripts.covid.knm_service import KNMClientFromExtension class RawSampleListColumns(object): @@ -26,7 +26,7 @@ def execute(self): ordering_org = self.context.current_step.udf_ordering_organization except AttributeError: self.usage_error("You must select an ordering organization") - client = KNMClient(self) + client = KNMClientFromExtension(self) raw_sample_list = RawSampleListFile.create_from_context(self.context) @@ -56,7 +56,7 @@ def execute(self): self.context.file_service.upload( validated_sample_list.FILE_HANDLE, file_name, validated_sample_list_content, self.context.file_service.FILE_PREFIX_NONE) - if len(unregistered) > 0 : + if len(unregistered) > 0: self.usage_warning("The following sample are unregistered '{}'. Press '{}' " "to change the 'Status' to anonymous" "".format(unregistered, BUTTON_TEXT_ASSIGN_UNREGISTERED_TO_ANONYMOUS)) diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/validate_sample_creation_list.py b/clarity-ext-scripts/clarity_ext_scripts/covid/validate_sample_creation_list.py index 3302384..bc099bd 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/validate_sample_creation_list.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/validate_sample_creation_list.py @@ -3,7 +3,7 @@ from datetime import datetime from clarity_ext.extensions import GeneralExtension from clarity_ext_scripts.covid.controls import controls_barcode_generator -from clarity_ext_scripts.covid.utils import KNMClient +from clarity_ext_scripts.covid.knm_service import KNMClientFromExtension from clarity_ext_scripts.covid.create_samples.common import ( BaseRawSampleListFile, ValidatedSampleListFile, BaseValidateRawSampleListExtension, BUTTON_TEXT_ASSIGN_UNREGISTERED_TO_ANONYMOUS @@ -96,7 +96,7 @@ def execute(self): # 2. Create an API client # Make sure that there is a config at ~/.config/clarity-ext/clarity-ext.config - client = KNMClient(self) + client = KNMClientFromExtension(self) # 3. Read the raw sample list. raw_sample_list = RawSampleListFile.create_from_context(self.context) @@ -148,7 +148,7 @@ def execute(self): self.context.file_service.upload( "Validated sample list", file_name, validated_sample_list_content, self.context.file_service.FILE_PREFIX_NONE) - if len(unregistered) > 0 : + if len(unregistered) > 0: self.usage_warning("The following sample are unregistered '{}'. Press '{}' " "to change the 'Status' to anonymous" "".format(unregistered, BUTTON_TEXT_ASSIGN_UNREGISTERED_TO_ANONYMOUS)) From 4556ca447daf854a23a934e58bbdf42fa84110ae Mon Sep 17 00:00:00 2001 From: Steinar Sturlaugsson Date: Fri, 29 May 2020 13:58:26 +0200 Subject: [PATCH 04/14] SmiNet and KNM integration --- .../covid/discard_samples.py | 7 +- .../clarity_ext_scripts/covid/knm_service.py | 78 +++++++++ .../covid/knm_sminet_service.py | 157 +++++++++++++++++ .../covid/partner_api_client.py | 20 ++- .../covid/sminet_settings.py | 35 ++++ clarity-ext-scripts/requirements.txt | 1 + .../tests/integration/test_knm_sminet.py | 159 ++++++++++++++++++ 7 files changed, 451 insertions(+), 6 deletions(-) create mode 100644 clarity-ext-scripts/clarity_ext_scripts/covid/knm_service.py create mode 100644 clarity-ext-scripts/clarity_ext_scripts/covid/knm_sminet_service.py create mode 100644 clarity-ext-scripts/clarity_ext_scripts/covid/sminet_settings.py create mode 100644 clarity-ext-scripts/tests/integration/test_knm_sminet.py diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/discard_samples.py b/clarity-ext-scripts/clarity_ext_scripts/covid/discard_samples.py index de52b2a..5b1a123 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/discard_samples.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/discard_samples.py @@ -1,11 +1,8 @@ import logging from datetime import datetime -from clarity_ext.extensions import GeneralExtension from clarity_ext_scripts.covid.partner_api_client import ( - PartnerAPIV7Client, TESTING_ORG, ORG_URI_BY_NAME, COVID_RESPONSE_FAILED, - PartnerClientAPIException) -from clarity_ext_scripts.covid.rtpcr_analysis_service import FAILED_STATES -from clarity_ext_scripts.covid.controls import Controls + TESTING_ORG, ORG_URI_BY_NAME, COVID_RESPONSE_FAILED, + PartnerClientAPIException) from clarity_ext_scripts.covid.utils import CtmrCovidSubstanceInfo, KNMClientFromExtension from clarity_ext_scripts.covid.import_samples import BaseCreateSamplesExtension diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/knm_service.py b/clarity-ext-scripts/clarity_ext_scripts/covid/knm_service.py new file mode 100644 index 0000000..17bc030 --- /dev/null +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/knm_service.py @@ -0,0 +1,78 @@ +from clarity_ext_scripts.covid.partner_api_client import PartnerAPIV7Client +from clarity_ext.utils import lazyprop + + +def KNMConfig(config): + """ + Creates config required for KNM from the clarity-ext config (which has more than that) + """ + return { + key: config[key] + for key in [ + "test_partner_base_url", + "test_partner_code_system_base_url", + "test_partner_user", + "test_partner_password" + ] + } + + +def KNMClientFromExtension(extension): + # A factory for a KnmClient from an extension + config = KNMConfig(extension.config) + return PartnerAPIV7Client(**config) + + +class KNMSampleAccessor(object): + """ + Describes a sample/analyte in the LIMS that has originally come via the KNM workflow + """ + + def __init__(self, org_uri, org_referral_code, date_arrival): + self.org_uri = org_uri + self.org_referral_code = org_referral_code + self.date_arrival = date_arrival + + @classmethod + def create_from_lims_sample(cls, sample): + pass + + +class ServiceRequestProvider(object): + """ + Represents a service request. Has methods to retrieve all data we require on it. Gives + a higher level abstraction of the api for readability. + """ + + def __init__(self, client, org_uri, org_referral_code): + self.client = client + self.org_uri = org_uri + self.org_referral_code = org_referral_code + + @lazyprop + def service_request(self): + # The service_request json response + return self.client.search_for_service_request(self.org_uri, self.org_referral_code) + + @lazyprop + def patient(self): + # The patient data corresponding with the service request + return self.client.get_by_reference(self.patient_ref) + + @lazyprop + def organization(self): + managing_organization = self.patient["managingOrganization"] + return self.client.get_by_reference(managing_organization["reference"]) + + @property + def patient_ref(self): + # A reference identifying the patient + return self.service_request["resource"]["subject"]["reference"] + + def __str__(self): + return "{}|{}".format(self.org_uri, self.org_referral_code) + + +class NoSupportedCountyCodeFound(Exception): + pass + diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/knm_sminet_service.py b/clarity-ext-scripts/clarity_ext_scripts/covid/knm_sminet_service.py new file mode 100644 index 0000000..fc6de66 --- /dev/null +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/knm_sminet_service.py @@ -0,0 +1,157 @@ +import codecs +import dateutil.parser +from datetime import datetime +from clarity_ext_scripts.covid.partner_api_client import PartnerAPIV7Client +from sminet_client import SmiNetClient, SmiNetConfig +from clarity_ext.utils import lazyprop +from sminet_client import (SampleInfo, ReferringClinic, Patient, Doctor, SmiNetConfig, NotificationType, + LabDiagnosisType, LabResult, Laboratory, SmiNetLabExport) +from sminet_client import Gender as SmiNetGender +from clarity_ext_scripts.covid.knm_service import KNMConfig, ServiceRequestProvider + + +class KNMSmiNetIntegrationService(object): + + MAP_GENDER_FROM_KNM_TO_SMINET = { + "male": SmiNetGender.MALE, + "female": SmiNetGender.FEMALE, + "other": SmiNetGender.UNKNOWN, + "unknown": SmiNetGender.UNKNOWN, + None: SmiNetGender.UNKNOWN, + } + + def __init__(self, config): + self.config = config + + knm_config = KNMConfig(config) + sminet_config = SmiNetConfig(**config) + + self.knm_client = PartnerAPIV7Client(**knm_config) + self.sminet_client = SmiNetClient(sminet_config) + + def get_county_from_organization(self, organization): + """ + Given the raw json from the KNM service, returns a county understood by SmiNet + """ + aliases = organization["alias"] + for alias in aliases: + county_code = alias.split("-")[0] + # TODO: THIS IS JUST FOR TEMPORARY TEST PURPOSES (waiting for knm fix) + if county_code == "SE": + county_code = "AB" + + if self.sminet_client.is_supported_county_code(county_code): + return county_code + raise NoSupportedCountyCodeFound( + "No supported county code found in alias list. Found: {}".format(aliases)) + + def create_sample_info(self, provider, sample): + sample_date_referral = provider.service_request["resource"]["authoredOn"] + # The date is on ISO8601 format: + sample_date_referral = dateutil.parser.isoparse(sample_date_referral) + + return SampleInfo(status=1, + sample_id=sample.org_referral_code, + sample_date_arrival=sample.date_arrival, + sample_date_referral=sample_date_referral, + sample_material="Svalg", + sample_free_text_referral="Anamnes: Personalprov") + + def create_referring_clinic(self, provider): + """Creates a referring clinic object from KNM data""" + referring_clinic_name = provider.patient["managingOrganization"]["display"] + referring_clinic_county = self.get_county_from_organization( + provider.organization) + referring_doctor = provider.service_request["resource"]["requester"]["display"] + + # TODO: We don't have an address for the referring clinic + return ReferringClinic(referring_clinic_name, "", + referring_clinic_county, Doctor(referring_doctor)) + + + def map_gender(self, knm_gender): + """Returns gender in the format required by SmiNet""" + pass + + def create_patient(self, provider): + """Creates a Patient object from KNM data""" + + def patient_identifier(): + try: + patient_identifier = provider.patient["identifier"] + except KeyError: + raise IntegrationError( + "Missing field 'identifier' on the patient resource for {}".format(provider)) + + if len(patient_identifier) == 0: + raise IntegrationError( + "Field 'identifier' is empty on patient resource for {}".format(provider)) + + try: + return patient_identifier[0]["value"] + except: + raise IntegrationError( + "First entry in 'identifier' doesn't have a value key for {}".format(provider)) + + def patient_gender(): + gender = provider.patient["gender"] + try: + return self.MAP_GENDER_FROM_KNM_TO_SMINET[gender] + except KeyError: + raise IntegrationError( + "Unexpected value for key 'gender' ({}) for {}".format(gender, provider)) + + def patient_name(): + """ + The name of the patient. Returns an empty string if we don't have the expected data + """ + name = provider.patient["name"] + + if len(name) == 0: + return "" + + name = name[0] + + try: + return name["text"] + except KeyError: + return "" + + return Patient(patient_identifier(), + patient_gender(), + patient_name(), + None) + + def get_sminet_export(self, sample): + """ + Returns a SmiNetLabExport from a sample + + :sample: A KNMSampleAccessor object + """ + + provider = ServiceRequestProvider( + self.knm_client, sample.org_uri, sample.org_referral_code) + + sample_info = self.create_sample_info(provider, sample) + print(sample_info) + reporting_doctor = Doctor("Lars Engstrand") + print(reporting_doctor) + referring_clinic = self.create_referring_clinic(provider) + print(referring_clinic) + patient = self.create_patient(provider) + lab_result = LabResult("C", + LabDiagnosisType("SCOV2", "SARS-CoV-2 (covid-19)")) + + notification = NotificationType(sample_info, reporting_doctor, referring_clinic, patient, + lab_result) + laboratory = Laboratory(91, "National Pandemic Center at KI") + + return SmiNetLabExport(datetime.utcnow(), laboratory, notification) + + def export_to_sminet(self, sample): + export = self.get_sminet_export(sample) + self.sminet_client.create(export, export.notification.sample_info.sample_id) + + +class IntegrationError(Exception): + pass diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/partner_api_client.py b/clarity-ext-scripts/clarity_ext_scripts/covid/partner_api_client.py index 0af08a3..fc75ab5 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/partner_api_client.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/partner_api_client.py @@ -1,4 +1,3 @@ - import base64 from datetime import datetime from luhn import verify as mod10verify @@ -400,6 +399,25 @@ def post_diagnosis_report(self, service_request_id, diagnosis_result, analysis_r log.error(e.message) raise e + def get_by_reference(self, ref): + """ + Get's a resource by reference, such as Organization/123 or Patient/345 + + :ref: The reference, e.g. Patient/123 + """ + url = "{}/{}".format(self._base_url, ref) + headers = self._generate_headers() + response = self._session.get(url=url, headers=headers) + if response.status_code != 200: + print(response.text) + raise PartnerClientAPIException( + "Couldn't get resource '{}', status code: {}".format( + url, response.status_code)) + return response.json() + + def get_org_uri_by_name(self, name): + return ORG_URI_BY_NAME[name] + def _create_payload(self, service_request_id, diagnosis_result, analysis_results): # TODO Need to think about if this needs refactoring later, to more easily support multiple # analysis types. This is going to be rather unwieldy to add more as it is implemented diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/sminet_settings.py b/clarity-ext-scripts/clarity_ext_scripts/covid/sminet_settings.py new file mode 100644 index 0000000..18aabd3 --- /dev/null +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/sminet_settings.py @@ -0,0 +1,35 @@ +from datetime import datetime +from sminet_client import (SmiNetConfig, NotificationType, LabResult, + Laboratory, SmiNetLabExport, LabDiagnosisType) + + +def create_scov2_positive_lab_result(): + lab_diagnosis = LabDiagnosisType("SCOV2", "SARS-CoV-2 (covid-19)") + return LabResult("C", lab_diagnosis) + + +def create_covid_request(sample_info, referring_clinic, patient, reporting_doctor, + laboratory, created=None): + """ + Creates a request to SmiNet that's valid for the Covid project. + + :sample_info: An object created with the SampleInfo factory + :referring_clinic: An object created with the ReferringClinic factory + :patient: An object created with the Patient factory + """ + + if not created: + created = datetime.now() + + notification = NotificationType(sample_info, reporting_doctor, referring_clinic, patient, + create_scov2_positive_lab_result()) + + contract = SmiNetLabExport(created, laboratory, notification) + return contract + + +def get_sminet_config(): + """ + Gets a configuration object for the sminet client fetched from the clarity-ext config file + """ + return SmiNetConfig.create_from_search_paths(["~/.config/clarity-ext/clarity-ext.config"]) diff --git a/clarity-ext-scripts/requirements.txt b/clarity-ext-scripts/requirements.txt index eb3e87d..d069817 100644 --- a/clarity-ext-scripts/requirements.txt +++ b/clarity-ext-scripts/requirements.txt @@ -4,3 +4,4 @@ pytest==4.6.9 requests[socks]==2.23.0 retry==0.9.2 suds==0.4 +python-dateutil==2.8.1 diff --git a/clarity-ext-scripts/tests/integration/test_knm_sminet.py b/clarity-ext-scripts/tests/integration/test_knm_sminet.py new file mode 100644 index 0000000..9c16823 --- /dev/null +++ b/clarity-ext-scripts/tests/integration/test_knm_sminet.py @@ -0,0 +1,159 @@ +import os +from lxml import etree as ET +from uuid import uuid4 +import random +import string +import yaml +import pytest +from datetime import datetime +from sminet_client import (SampleInfo, ReferringClinic, Patient, Doctor, + SmiNetClient, SmiNetValidationError, SmiNetConfig, Laboratory) + +from clarity_ext_scripts.covid.sminet_settings import get_sminet_config, create_covid_request +from clarity_ext_scripts.covid.knm_service import KNMConfig +from clarity_ext_scripts.covid.knm_sminet_service import KNMSmiNetIntegrationService +from clarity_ext_scripts.covid.partner_api_client import ( + PartnerAPIV7Client, KARLSSON_AND_NOVAK, ORG_URI_BY_NAME) +from clarity_ext.cli import load_config + + +""" +Tests the SmiNet/KNM integration +""" + +try: + config = load_config() +except ConfigNotFoundError: + pytest.xfail("This test requires a sminet_client configuration file") + + +def generate_valid_contract_with_random_sample_id(): + """ + Returns a tuple of the parameters required to upload sample info via client.create + """ + constant_date = datetime(2020, 4, 30) + request_created = datetime(2020, 5, 18, 18, 11, 6) + timestamp = datetime.now().strftime("%y%m%dT%H%M%S") + prefix = "{}-".format(timestamp) + rnd = "".join(random.choice(string.ascii_uppercase + string.digits) + for _ in range(25 - len(prefix))) + sample_id = prefix + rnd + + sample_info = SampleInfo(status=1, sample_id=sample_id, sample_date_arrival=constant_date, + sample_date_referral=constant_date, sample_material="Svalg", + sample_free_text_referral="Extra info") + referring_clinic = ReferringClinic( + "Clinic name", "", "C", Doctor("Some doctor")) + patient = Patient("121212-1212", "k", "Tolvan Tolvansson", 23) + reporting_doctor = Doctor("Reporting Doctor") + laboratory = Laboratory(91, "National Pandemic Center at KI") + return create_covid_request(sample_info, referring_clinic, patient, reporting_doctor, laboratory) + + +@pytest.mark.now() +def test_can_create_request(): + """ + NOTE: We work against a whitelist on a jump server. Information available from team lead. + NOTE: Reads config from your clarity-ext config which should point to the test environment. + If sminet config is not available, the test is ignored. This is because the test + can't run on the build server because it's not whitelisted at the moment. + """ + + from clarity_ext_scripts.covid.knm_service import KNMSampleAccessor + + integration = KNMSmiNetIntegrationService(config) + org_uri = ORG_URI_BY_NAME[KARLSSON_AND_NOVAK] + sample = KNMSampleAccessor(org_uri, "5236417647", datetime.now()) + integration.export_to_sminet(sample) + return + + #7224014063 + print(organization) + print(patient) + print("INFO") + print(referring_clinic_name) + + return + + print(req) + client = SmiNetClient(config) + + # export = generate_valid_contract_with_random_sample_id() + # file_name = export.notification.sample_info.sample_id + # client.create(export, file_name) + + sminet_client = get_sminet_client() + + print(org_uri) + + #sample_date_referral = req. + from pprint import pprint + pprint(req) + resource = req["resource"] + + ## Build SampleInfo + sample_id = resource["id"] + # TODO: date-received on the submitted sample + sample_date_arrival = datetime.datetime(2020, 1, 1) + sample_date_referral = resource["authoredOn"] # ISO-format + sample_date_referral = knm_client.parse_date(sample_date_referral) + sample_info = SampleInfo( + status=1, + sample_id=sample_id, + sample_date_arrival=sample_date_arrival, + sample_date_referral=sample_date_referral, + sample_material="Svalg", + sample_free_text_referral="Anamnes: Personalprov") + print(sample_info) + + ## Build ReferringClinic: + # referringDoctor: {ServiceRequest.requester.display} + # resource["requester"]["display"] TODO: is None now! + + ## Build Patient object + # + patient_reference = resource["subject"]["reference"] + patient_json = knm_client.get_by_reference(patient_reference) + + patient_organization_ref = patient_json["managingOrganization"]["reference"] + organization_json = knm_client.get_by_reference(patient_organization_ref) + + patient_sex = patient_json["gender"] + + print("Patient") + print(patient_json) + print(organization_json) + print(patient_sex) + + print(knm_client.get_by_reference("Organization/4")) + + # Patient + # patientId: {ServiceRequest.subject.reference -> Patient.identifier[0].value} + # patientSex: {ServiceRequest.subject.reference -> Patient.gender} + # patientName: {ServiceRequest.subject.reference -> Patient.name[0].text} + # patientAge: {(ServiceRequest.subject.reference -> Patient.identifier[0].value[8:] - NOW() )} + + # {u'managingOrganization': + # { + # u'display': u'Region V\xe4sterbotten - Personalprov regionen', u'reference': u'Organization/4-9'}, + # u'name': [{u'text': u'No one', u'given': [u'No'], u'family': u'one'}], + # u'resourceType': u'Patient', + # u'gender': u'unknown', + # u'address': [{u'country': u'SE'}], + # u'id': u'626' + # } + + # {u'resource': { + # u'code': {u'coding': [{u'code': u'covid19', + # u'system': u'http://uri.d-t.se/id/CodeSystem/cs-test-types'}], + # u'text': u'Covid-19'}, + # u'id': u'624', + # u'identifier': [{u'assigner': {u'display': u'Direkttest unique referral code'}, + # u'system': u'http://uri.d-t.se/id/Identifier/i-referral-code', + # u'value': u'3834043766'}], + # u'intent': u'original-order', + # u'requester': None, + # u'resourceType': u'ServiceRequest', + # u'status': u'active', + # u'subject': {u'reference': u'Patient/626'}}, + # u'search': {u'mode': u'match'}} From dc8756e83cfcd0529940dbde1476bdcc6d7085bd Mon Sep 17 00:00:00 2001 From: Steinar Sturlaugsson Date: Mon, 1 Jun 2020 13:10:49 +0200 Subject: [PATCH 05/14] PR fixes --- .../covid/import_samples.py | 2 +- .../covid/services/__init__.py | 0 .../covid/{ => services}/knm_service.py | 28 +++- .../{ => services}/knm_sminet_service.py | 78 ++++----- .../covid/services/sminet_service.py | 20 +++ .../covid/sminet_settings.py | 35 ---- .../covid/validate_discarded_samples.py | 2 +- .../tests/integration/test_knm_sminet.py | 150 ++---------------- sminet-client/sminet_client/__init__.py | 113 +++++++------ 9 files changed, 149 insertions(+), 279 deletions(-) create mode 100644 clarity-ext-scripts/clarity_ext_scripts/covid/services/__init__.py rename clarity-ext-scripts/clarity_ext_scripts/covid/{ => services}/knm_service.py (77%) rename clarity-ext-scripts/clarity_ext_scripts/covid/{ => services}/knm_sminet_service.py (63%) create mode 100644 clarity-ext-scripts/clarity_ext_scripts/covid/services/sminet_service.py delete mode 100644 clarity-ext-scripts/clarity_ext_scripts/covid/sminet_settings.py diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/import_samples.py b/clarity-ext-scripts/clarity_ext_scripts/covid/import_samples.py index 77e5f72..771b493 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/import_samples.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/import_samples.py @@ -1,6 +1,6 @@ from clarity_ext.domain import Container, Sample from clarity_ext_scripts.covid.controls import controls_barcode_generator, Controls -from clarity_ext_scripts.covid.knm_service import KNMClientFromExtension +from clarity_ext_scripts.covid.services.knm_service import KNMClientFromExtension from clarity_ext_scripts.covid.create_samples.common import BaseCreateSamplesExtension diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/services/__init__.py b/clarity-ext-scripts/clarity_ext_scripts/covid/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/knm_service.py b/clarity-ext-scripts/clarity_ext_scripts/covid/services/knm_service.py similarity index 77% rename from clarity-ext-scripts/clarity_ext_scripts/covid/knm_service.py rename to clarity-ext-scripts/clarity_ext_scripts/covid/services/knm_service.py index 17bc030..7ce2473 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/knm_service.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/services/knm_service.py @@ -2,6 +2,18 @@ from clarity_ext.utils import lazyprop +class NoSupportedCountyCodeFound(Exception): + pass + + +class KNMService(object): + NoSupportedCountyCodeFound = NoSupportedCountyCodeFound + + def __init__(self, config): + self.config = KNMConfig(config) + self.client = PartnerAPIV7Client(**self.config) + + def KNMConfig(config): """ Creates config required for KNM from the clarity-ext config (which has more than that) @@ -28,14 +40,21 @@ class KNMSampleAccessor(object): Describes a sample/analyte in the LIMS that has originally come via the KNM workflow """ - def __init__(self, org_uri, org_referral_code, date_arrival): + def __init__(self, org_uri, org_referral_code, date_arrival, material): + """ + :org_uri: The organization URI as defined by KNM + :org_referral_code: The referral code within the organization + :date_arrival: The date the sample was added to the LIMS + :material: The material, one of the constants in SampleMaterialType + """ self.org_uri = org_uri self.org_referral_code = org_referral_code self.date_arrival = date_arrival + self.material = material @classmethod def create_from_lims_sample(cls, sample): - pass + raise NotImplementedError() class ServiceRequestProvider(object): @@ -71,8 +90,3 @@ def patient_ref(self): def __str__(self): return "{}|{}".format(self.org_uri, self.org_referral_code) - - -class NoSupportedCountyCodeFound(Exception): - pass - diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/knm_sminet_service.py b/clarity-ext-scripts/clarity_ext_scripts/covid/services/knm_sminet_service.py similarity index 63% rename from clarity-ext-scripts/clarity_ext_scripts/covid/knm_sminet_service.py rename to clarity-ext-scripts/clarity_ext_scripts/covid/services/knm_sminet_service.py index fc6de66..abd584a 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/knm_sminet_service.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/services/knm_sminet_service.py @@ -1,13 +1,9 @@ -import codecs import dateutil.parser from datetime import datetime -from clarity_ext_scripts.covid.partner_api_client import PartnerAPIV7Client -from sminet_client import SmiNetClient, SmiNetConfig -from clarity_ext.utils import lazyprop -from sminet_client import (SampleInfo, ReferringClinic, Patient, Doctor, SmiNetConfig, NotificationType, - LabDiagnosisType, LabResult, Laboratory, SmiNetLabExport) +from sminet_client import (SampleInfo, ReferringClinic, Patient, Doctor, + SmiNetLabExport, StatusType, Notification) from sminet_client import Gender as SmiNetGender -from clarity_ext_scripts.covid.knm_service import KNMConfig, ServiceRequestProvider +from clarity_ext_scripts.covid.services.knm_service import ServiceRequestProvider class KNMSmiNetIntegrationService(object): @@ -20,14 +16,10 @@ class KNMSmiNetIntegrationService(object): None: SmiNetGender.UNKNOWN, } - def __init__(self, config): + def __init__(self, config, knm_service, sminet_service): self.config = config - - knm_config = KNMConfig(config) - sminet_config = SmiNetConfig(**config) - - self.knm_client = PartnerAPIV7Client(**knm_config) - self.sminet_client = SmiNetClient(sminet_config) + self.sminet_service = sminet_service + self.knm_service = knm_service def get_county_from_organization(self, organization): """ @@ -36,26 +28,22 @@ def get_county_from_organization(self, organization): aliases = organization["alias"] for alias in aliases: county_code = alias.split("-")[0] - # TODO: THIS IS JUST FOR TEMPORARY TEST PURPOSES (waiting for knm fix) - if county_code == "SE": - county_code = "AB" - - if self.sminet_client.is_supported_county_code(county_code): + if self.sminet_service.client.is_supported_county_code(county_code): return county_code - raise NoSupportedCountyCodeFound( + raise self.knm_service.NoSupportedCountyCodeFound( "No supported county code found in alias list. Found: {}".format(aliases)) - def create_sample_info(self, provider, sample): + def create_sample_info(self, provider, sample, free_text): sample_date_referral = provider.service_request["resource"]["authoredOn"] # The date is on ISO8601 format: sample_date_referral = dateutil.parser.isoparse(sample_date_referral) - return SampleInfo(status=1, + return SampleInfo(status=StatusType.FINAL_RESPONSE, sample_id=sample.org_referral_code, sample_date_arrival=sample.date_arrival, sample_date_referral=sample_date_referral, - sample_material="Svalg", - sample_free_text_referral="Anamnes: Personalprov") + sample_material=sample.material, + sample_free_text_referral=free_text) def create_referring_clinic(self, provider): """Creates a referring clinic object from KNM data""" @@ -68,11 +56,6 @@ def create_referring_clinic(self, provider): return ReferringClinic(referring_clinic_name, "", referring_clinic_county, Doctor(referring_doctor)) - - def map_gender(self, knm_gender): - """Returns gender in the format required by SmiNet""" - pass - def create_patient(self, provider): """Creates a Patient object from KNM data""" @@ -89,7 +72,7 @@ def patient_identifier(): try: return patient_identifier[0]["value"] - except: + except KeyError: raise IntegrationError( "First entry in 'identifier' doesn't have a value key for {}".format(provider)) @@ -122,35 +105,32 @@ def patient_name(): patient_name(), None) - def get_sminet_export(self, sample): + def export_to_sminet(self, sample, doctor_name, lab_result, sample_free_text): """ - Returns a SmiNetLabExport from a sample + Exports a SmiNetLabExport based on a sample :sample: A KNMSampleAccessor object + :doctor_name: Name of the doctor + :lab_result: A LabResult entry (for convenience one can use those created by SmiNetService) """ - + # Generate export: provider = ServiceRequestProvider( - self.knm_client, sample.org_uri, sample.org_referral_code) + self.knm_service.client, sample.org_uri, sample.org_referral_code) - sample_info = self.create_sample_info(provider, sample) - print(sample_info) - reporting_doctor = Doctor("Lars Engstrand") - print(reporting_doctor) + sample_info = self.create_sample_info( + provider, sample, sample_free_text) + reporting_doctor = Doctor(doctor_name) referring_clinic = self.create_referring_clinic(provider) - print(referring_clinic) patient = self.create_patient(provider) - lab_result = LabResult("C", - LabDiagnosisType("SCOV2", "SARS-CoV-2 (covid-19)")) - - notification = NotificationType(sample_info, reporting_doctor, referring_clinic, patient, - lab_result) - laboratory = Laboratory(91, "National Pandemic Center at KI") + notification = Notification(sample_info, reporting_doctor, referring_clinic, patient, + lab_result) + laboratory = self.sminet_service.get_laboratory() - return SmiNetLabExport(datetime.utcnow(), laboratory, notification) + export = SmiNetLabExport(datetime.now(), laboratory, notification) - def export_to_sminet(self, sample): - export = self.get_sminet_export(sample) - self.sminet_client.create(export, export.notification.sample_info.sample_id) + # Send it + self.sminet_service.client.create( + export, export.notification.sample_info.sample_id) class IntegrationError(Exception): diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/services/sminet_service.py b/clarity-ext-scripts/clarity_ext_scripts/covid/services/sminet_service.py new file mode 100644 index 0000000..d5d4ecb --- /dev/null +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/services/sminet_service.py @@ -0,0 +1,20 @@ +from sminet_client import (SmiNetConfig, LabResult, + Laboratory, LabDiagnosisType, SmiNetClient) + + +class SmiNetService(object): + + def __init__(self, config): + self.config = SmiNetConfig(**config) + self.client = SmiNetClient(self.config) + + def get_laboratory(self): + """ + Retrieves the configured laboratory + """ + return Laboratory(self.config.lab_number, self.config.lab_name) + + @staticmethod + def create_scov2_positive_lab_result(): + lab_diagnosis = LabDiagnosisType("SCOV2", "SARS-CoV-2 (covid-19)") + return LabResult("C", lab_diagnosis) diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/sminet_settings.py b/clarity-ext-scripts/clarity_ext_scripts/covid/sminet_settings.py deleted file mode 100644 index 18aabd3..0000000 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/sminet_settings.py +++ /dev/null @@ -1,35 +0,0 @@ -from datetime import datetime -from sminet_client import (SmiNetConfig, NotificationType, LabResult, - Laboratory, SmiNetLabExport, LabDiagnosisType) - - -def create_scov2_positive_lab_result(): - lab_diagnosis = LabDiagnosisType("SCOV2", "SARS-CoV-2 (covid-19)") - return LabResult("C", lab_diagnosis) - - -def create_covid_request(sample_info, referring_clinic, patient, reporting_doctor, - laboratory, created=None): - """ - Creates a request to SmiNet that's valid for the Covid project. - - :sample_info: An object created with the SampleInfo factory - :referring_clinic: An object created with the ReferringClinic factory - :patient: An object created with the Patient factory - """ - - if not created: - created = datetime.now() - - notification = NotificationType(sample_info, reporting_doctor, referring_clinic, patient, - create_scov2_positive_lab_result()) - - contract = SmiNetLabExport(created, laboratory, notification) - return contract - - -def get_sminet_config(): - """ - Gets a configuration object for the sminet client fetched from the clarity-ext config file - """ - return SmiNetConfig.create_from_search_paths(["~/.config/clarity-ext/clarity-ext.config"]) diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/validate_discarded_samples.py b/clarity-ext-scripts/clarity_ext_scripts/covid/validate_discarded_samples.py index 2fc294f..132762b 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/validate_discarded_samples.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/validate_discarded_samples.py @@ -2,7 +2,7 @@ from clarity_ext_scripts.covid.create_samples.common import ( BaseValidateRawSampleListExtension, BaseRawSampleListFile, BUTTON_TEXT_ASSIGN_UNREGISTERED_TO_ANONYMOUS) -from clarity_ext_scripts.covid.knm_service import KNMClientFromExtension +from clarity_ext_scripts.covid.services.knm_service import KNMClientFromExtension class RawSampleListColumns(object): diff --git a/clarity-ext-scripts/tests/integration/test_knm_sminet.py b/clarity-ext-scripts/tests/integration/test_knm_sminet.py index 9c16823..55c04b6 100644 --- a/clarity-ext-scripts/tests/integration/test_knm_sminet.py +++ b/clarity-ext-scripts/tests/integration/test_knm_sminet.py @@ -1,20 +1,10 @@ -import os -from lxml import etree as ET -from uuid import uuid4 -import random -import string -import yaml import pytest from datetime import datetime -from sminet_client import (SampleInfo, ReferringClinic, Patient, Doctor, - SmiNetClient, SmiNetValidationError, SmiNetConfig, Laboratory) - -from clarity_ext_scripts.covid.sminet_settings import get_sminet_config, create_covid_request -from clarity_ext_scripts.covid.knm_service import KNMConfig -from clarity_ext_scripts.covid.knm_sminet_service import KNMSmiNetIntegrationService -from clarity_ext_scripts.covid.partner_api_client import ( - PartnerAPIV7Client, KARLSSON_AND_NOVAK, ORG_URI_BY_NAME) +from clarity_ext_scripts.covid.services.sminet_service import SmiNetService +from clarity_ext_scripts.covid.services.knm_sminet_service import KNMSmiNetIntegrationService +from clarity_ext_scripts.covid.partner_api_client import KARLSSON_AND_NOVAK, ORG_URI_BY_NAME from clarity_ext.cli import load_config +from clarity_ext_scripts.covid.services.knm_service import KNMSampleAccessor, KNMService """ @@ -23,34 +13,10 @@ try: config = load_config() -except ConfigNotFoundError: +except IOError: pytest.xfail("This test requires a sminet_client configuration file") -def generate_valid_contract_with_random_sample_id(): - """ - Returns a tuple of the parameters required to upload sample info via client.create - """ - constant_date = datetime(2020, 4, 30) - request_created = datetime(2020, 5, 18, 18, 11, 6) - timestamp = datetime.now().strftime("%y%m%dT%H%M%S") - prefix = "{}-".format(timestamp) - rnd = "".join(random.choice(string.ascii_uppercase + string.digits) - for _ in range(25 - len(prefix))) - sample_id = prefix + rnd - - sample_info = SampleInfo(status=1, sample_id=sample_id, sample_date_arrival=constant_date, - sample_date_referral=constant_date, sample_material="Svalg", - sample_free_text_referral="Extra info") - referring_clinic = ReferringClinic( - "Clinic name", "", "C", Doctor("Some doctor")) - patient = Patient("121212-1212", "k", "Tolvan Tolvansson", 23) - reporting_doctor = Doctor("Reporting Doctor") - laboratory = Laboratory(91, "National Pandemic Center at KI") - return create_covid_request(sample_info, referring_clinic, patient, reporting_doctor, laboratory) - - -@pytest.mark.now() def test_can_create_request(): """ NOTE: We work against a whitelist on a jump server. Information available from team lead. @@ -59,101 +25,15 @@ def test_can_create_request(): can't run on the build server because it's not whitelisted at the moment. """ - from clarity_ext_scripts.covid.knm_service import KNMSampleAccessor - - integration = KNMSmiNetIntegrationService(config) + constant_date = datetime(2020, 4, 30) + knm_service = KNMService(config) + sminet_service = SmiNetService(config) + integration = KNMSmiNetIntegrationService( + config, knm_service, sminet_service) org_uri = ORG_URI_BY_NAME[KARLSSON_AND_NOVAK] - sample = KNMSampleAccessor(org_uri, "5236417647", datetime.now()) - integration.export_to_sminet(sample) - return - - #7224014063 - print(organization) - print(patient) - print("INFO") - print(referring_clinic_name) - - return - - print(req) - client = SmiNetClient(config) - - # export = generate_valid_contract_with_random_sample_id() - # file_name = export.notification.sample_info.sample_id - # client.create(export, file_name) - - sminet_client = get_sminet_client() - - print(org_uri) - - #sample_date_referral = req. - from pprint import pprint - pprint(req) - resource = req["resource"] - - ## Build SampleInfo - sample_id = resource["id"] - # TODO: date-received on the submitted sample - sample_date_arrival = datetime.datetime(2020, 1, 1) - sample_date_referral = resource["authoredOn"] # ISO-format - sample_date_referral = knm_client.parse_date(sample_date_referral) - sample_info = SampleInfo( - status=1, - sample_id=sample_id, - sample_date_arrival=sample_date_arrival, - sample_date_referral=sample_date_referral, - sample_material="Svalg", - sample_free_text_referral="Anamnes: Personalprov") - print(sample_info) - - ## Build ReferringClinic: - # referringDoctor: {ServiceRequest.requester.display} - # resource["requester"]["display"] TODO: is None now! - - ## Build Patient object - # - patient_reference = resource["subject"]["reference"] - patient_json = knm_client.get_by_reference(patient_reference) - - patient_organization_ref = patient_json["managingOrganization"]["reference"] - organization_json = knm_client.get_by_reference(patient_organization_ref) - - patient_sex = patient_json["gender"] - - print("Patient") - print(patient_json) - print(organization_json) - print(patient_sex) - - print(knm_client.get_by_reference("Organization/4")) - - # Patient - # patientId: {ServiceRequest.subject.reference -> Patient.identifier[0].value} - # patientSex: {ServiceRequest.subject.reference -> Patient.gender} - # patientName: {ServiceRequest.subject.reference -> Patient.name[0].text} - # patientAge: {(ServiceRequest.subject.reference -> Patient.identifier[0].value[8:] - NOW() )} - - # {u'managingOrganization': - # { - # u'display': u'Region V\xe4sterbotten - Personalprov regionen', u'reference': u'Organization/4-9'}, - # u'name': [{u'text': u'No one', u'given': [u'No'], u'family': u'one'}], - # u'resourceType': u'Patient', - # u'gender': u'unknown', - # u'address': [{u'country': u'SE'}], - # u'id': u'626' - # } + sample = KNMSampleAccessor(org_uri, "5236417647", constant_date, "Svalg") + sample_free_text = "Anamnes: Personalprov" - # {u'resource': { - # u'code': {u'coding': [{u'code': u'covid19', - # u'system': u'http://uri.d-t.se/id/CodeSystem/cs-test-types'}], - # u'text': u'Covid-19'}, - # u'id': u'624', - # u'identifier': [{u'assigner': {u'display': u'Direkttest unique referral code'}, - # u'system': u'http://uri.d-t.se/id/Identifier/i-referral-code', - # u'value': u'3834043766'}], - # u'intent': u'original-order', - # u'requester': None, - # u'resourceType': u'ServiceRequest', - # u'status': u'active', - # u'subject': {u'reference': u'Patient/626'}}, - # u'search': {u'mode': u'match'}} + lab_result = SmiNetService.create_scov2_positive_lab_result() + integration.export_to_sminet( + sample, "Lars Engstrand", lab_result, sample_free_text) diff --git a/sminet-client/sminet_client/__init__.py b/sminet-client/sminet_client/__init__.py index 46ee453..e1f160e 100644 --- a/sminet-client/sminet_client/__init__.py +++ b/sminet-client/sminet_client/__init__.py @@ -2,8 +2,6 @@ import os import yaml -import datetime -import requests import logging import codecs import base64 @@ -240,24 +238,29 @@ def Version(version_number): # NOTE: These are our translations of the status in the xsd StatusType documentation, which is in # Swedish. Might be inaccurate. -STATUS_FINAL_RESPONSE = 1 -STATUS_COMPLEMENTARY_DATA_PENDING = 2 -STATUS_REVOCATION_OF_PREVIOUS_REPORT = 3 -STATUS_COMPLEMENTARY_DATA = 4 -def StatusType(status): +class StatusType(object): """ Original: Avser en anmälans status. Förklaring: 1. Slutsvar. 2. Komplettering kommer. 3. Makulering av en tidigare anmälan. 4. Komplettering av en tidigare anmälan. """ - if status not in [STATUS_FINAL_RESPONSE, - STATUS_COMPLEMENTARY_DATA_PENDING, - STATUS_REVOCATION_OF_PREVIOUS_REPORT, - STATUS_COMPLEMENTARY_DATA]: - raise SmiNetValidationError("The status {} is not in the list of accepted statuses" - .format(status)) - return status + + FINAL_RESPONSE = 1 + COMPLEMENTARY_DATA_PENDING = 2 + REVOCATION_OF_PREVIOUS_REPORT = 3 + COMPLEMENTARY_DATA = 4 + + ALL = [FINAL_RESPONSE, + COMPLEMENTARY_DATA_PENDING, + REVOCATION_OF_PREVIOUS_REPORT, + COMPLEMENTARY_DATA] + + def __init__(self, status): + if status not in self.ALL: + raise SmiNetValidationError("The status {} is not in the list of accepted statuses" + .format(status)) + self.value = status def SampleMaterialType(material_type): @@ -390,7 +393,7 @@ def __init__(self, def to_element(self, element_name="sampleInfo"): element = ET.Element(element_name) - add_child(element, "status", self.status) + add_child(element, "status", self.status.value) add_child(element, "sampleNumber", self.sample_id) add_child(element, "sampleDateArrival", self.sample_date_arrival) add_child(element, "sampleDateReferral", self.sample_date_referral) @@ -402,7 +405,40 @@ def to_element(self, element_name="sampleInfo"): return element -class NotificationType(SmiNetComplexType): +class Patient(SmiNetComplexType): + """ + Information för att identifiera patienten. + """ + + def __init__(self, patient_id, patient_sex, patient_name=None, patient_age=None): + """ + :patient_id: Tillåtna typer är personnummer, rikskod och reservkod + :patient_sex: Patientens kön + :patient_name: Patientens namn + :patient_age: Patientens ålder + """ + self.patient_id = IdType(patient_id) + self.patient_sex = SexType(patient_sex) + self.patient_name = LimitedString(patient_name) + self.patient_age = NonNegativeInteger( + patient_age) if patient_age else None + + def to_element(self, element_name="patient"): + element = ET.Element(element_name) + add_child(element, "patientId", self.patient_id) + add_child(element, "patientSex", self.patient_sex) + add_child(element, "patientName", self.patient_name) + add_child(element, "patientAge", self.patient_age) + return element + + +class Notification(SmiNetComplexType): + + SampleInfo = SampleInfo + Doctor = Doctor + ReferringClinic = ReferringClinic + Patient = Patient + def __init__(self, sample_info, reporting_doctor, referring_clinic, patient, lab_result): """ :sample_info: Översiktlig information om provet. @@ -465,9 +501,9 @@ class LabResult(SmiNetComplexType): def __init__(self, diagnostic_method, lab_diagnosis): """ :diagnostic_method: Observera att det är SmiNet-koden för diagnostisk metod som - ska användas. + ska användas. :lab_diagnosis: Viktig typningsinformation om provet, som till exempel eventuell - typningsinformation om sådan finns. + typningsinformation om sådan finns. """ self.diagnostic_method = DiagnosticMethod(diagnostic_method) @@ -481,33 +517,6 @@ def to_element(self, element_name="labResult"): return element -class Patient(SmiNetComplexType): - """ - Information för att identifiera patienten. - """ - - def __init__(self, patient_id, patient_sex, patient_name=None, patient_age=None): - """ - :patient_id: Tillåtna typer är personnummer, rikskod och reservkod - :patient_sex: Patientens kön - :patient_name: Patientens namn - :patient_age: Patientens ålder - """ - self.patient_id = IdType(patient_id) - self.patient_sex = SexType(patient_sex) - self.patient_name = LimitedString(patient_name) - self.patient_age = NonNegativeInteger( - patient_age) if patient_age else None - - def to_element(self, element_name="patient"): - element = ET.Element(element_name) - add_child(element, "patientId", self.patient_id) - add_child(element, "patientSex", self.patient_sex) - add_child(element, "patientName", self.patient_name) - add_child(element, "patientAge", self.patient_age) - return element - - class SmiNetLabExport(): """ Creates a validated lab export as XML that is acceptable for the SmiNet endpoint. @@ -518,6 +527,10 @@ class SmiNetLabExport(): Varje exportfil (XML-fil) måste innehålla ett och endast ett sådant element. """ + # For convenience, all the types that make up an export are defined here: + Laboratory = Laboratory + Notification = Notification + def __init__(self, created, laboratory, notification): self.created = created self.laboratory = laboratory @@ -597,12 +610,10 @@ def create(self, sminet_lab_export, file_name): """ Creates the entry in SmiNet - :export: An instance of SmiNetLabExport - :file_name: Name of the file in SmiNet's database + :export: An instance of SmiNetLabExport + :file_name: Name of the file in SmiNet's database """ logger.info("Creating a request at SmiNet") - headers = {"Content-Type": "application/xml"} - doc = sminet_lab_export.to_document(self.xsd_url) self._send_file(doc, file_name) @@ -664,8 +675,8 @@ def __init__(self, sminet_username, :sminet_username: The username :sminet_password: The password - :sminet_environment: The environment, e.g. SmiNetClient.SMINET_ENVIRONMENT_STAGE - :proxy: Proxy info if the service is only accessible via a proxy + :sminet_environment: The environment, e.g. SmiNetClient.SMINET_ENVIRONMENT_STAGE + :proxy: Proxy info if the service is only accessible via a proxy :lab_name: The name of your lab :lab_number: The name of your lab """ From d6c56dafdcd7c8bd9f3504144371cdefe4ec2206 Mon Sep 17 00:00:00 2001 From: Steinar Sturlaugsson Date: Mon, 1 Jun 2020 15:19:02 +0200 Subject: [PATCH 06/14] Ensure sminet-client tests can run --- sminet-client/setup.py | 2 +- sminet-client/sminet_client/__init__.py | 64 +++++++++++++++---- .../tests/integration/test_sminet_client.py | 33 +++++----- .../tests/unit/test_sminet_client.py | 30 +++++---- 4 files changed, 87 insertions(+), 42 deletions(-) diff --git a/sminet-client/setup.py b/sminet-client/setup.py index edfd6f5..e8a2721 100644 --- a/sminet-client/setup.py +++ b/sminet-client/setup.py @@ -4,7 +4,7 @@ name='sminet-client', version='1.0.0', packages=find_packages(exclude=['tests']), - requirements=['suds'], + install_requires=['suds', 'pyyaml', 'lxml'], zip_safe=False, platforms='any', classifiers=[ diff --git a/sminet-client/sminet_client/__init__.py b/sminet-client/sminet_client/__init__.py index e1f160e..5f11f77 100644 --- a/sminet-client/sminet_client/__init__.py +++ b/sminet-client/sminet_client/__init__.py @@ -263,22 +263,64 @@ def __init__(self, status): self.value = status -def SampleMaterialType(material_type): +class SampleMaterialType(object): """ Avser undersökningsmatrial. (Notera att det är tillåtet att byta ut 'å' och 'ä' mot 'a' samt att 'ö' kan bytas ut mot 'o'. Detta alternativ finns om det skulle bli problem med valideringen pga konstigheter i teckentabeller.) """ - supported = ["Annat" "Bio", "Blod", "Bronk", "Feces", "Likv", "Lymf", - "Nfary", "Nasa", "Pleur", "Sekr", "Serum", "Sput", "Svalg", "Sar", - "Urin", "VSK", "Led", "Perik", "Fost", "Asci", "Melor", "Uret", - "Rect", "Cerv", "VagSek", "Asp", "Blodsr", "Cervur", "Infart", "Nsp", - "Perin", "Poolat", "Saliv", "Vagur", "Ogon", "Oron"] - if material_type not in supported: - raise SmiNetValidationError( - "Material type not supported: {}".format(material_type)) - return material_type + OTHER = "Annat" + BIO = "Bio" + BLOD = "Blod" + BRONK = "Bronk" + FECES = "Feces" + LIKV = "Likv" + LYMF = "Lymf" + NFARY = "Nfary" + NASA = "Nasa" + PLEUR = "Pleur" + SEKR = "Sekr" + SERUM = "Serum" + SPUT = "Sput" + SVALG = "Svalg" + SAR = "Sar" + URIN = "Urin" + VSK = "VSK" + LED = "Led" + PERIK = "Perik" + FOST = "Fost" + ASCI = "Asci" + MELOR = "Melor" + URET = "Uret" + RECT = "Rect" + CERV = "Cerv" + VAGSEK = "VagSek" + ASP = "Asp" + BLODSR = "Blodsr" + CERVUR = "Cervur" + INFART = "Infart" + NSP = "Nsp" + PERIN = "Perin" + POOLAT = "Poolat" + SALIV = "Saliv" + VAGUR = "Vagur" + OGON = "Ogon" + ORON = "Oron" + + ALL = [OTHER, BIO, BLOD, BRONK, FECES, LIKV, + LYMF, NFARY, NASA, PLEUR, SEKR, SERUM, + SPUT, SVALG, SAR, URIN, VSK, LED, + PERIK, FOST, ASCI, MELOR, URET, RECT, + CERV, VAGSEK, ASP, BLODSR, CERVUR, INFART, + NSP, PERIN, POOLAT, SALIV, VAGUR, + OGON, ORON] + + def __init__(self, value): + if value not in self.ALL: + raise SmiNetValidationError( + "Material type not supported: {}".format(value)) + self.value = value ##### @@ -397,7 +439,7 @@ def to_element(self, element_name="sampleInfo"): add_child(element, "sampleNumber", self.sample_id) add_child(element, "sampleDateArrival", self.sample_date_arrival) add_child(element, "sampleDateReferral", self.sample_date_referral) - add_child(element, "sampleMaterial", self.sample_material) + add_child(element, "sampleMaterial", self.sample_material.value) add_child(element, "optionalReference", self.optional_reference) add_child(element, "sampleFreeTextLab", self.sample_free_text_lab) add_child(element, "sampleFreeTextReferral", diff --git a/sminet-client/tests/integration/test_sminet_client.py b/sminet-client/tests/integration/test_sminet_client.py index 6c89dce..9b4539f 100644 --- a/sminet-client/tests/integration/test_sminet_client.py +++ b/sminet-client/tests/integration/test_sminet_client.py @@ -1,12 +1,12 @@ -import os import string import random from datetime import datetime import lxml.etree as ET import pytest -from sminet_client import (SampleInfo, ReferringClinic, Patient, Doctor, SmiNetConfig, NotificationType, - LabDiagnosisType, LabResult, Laboratory, SmiNetLabExport, ConfigNotFoundError, - SmiNetClient, SmiNetValidationError) +from sminet_client import (SampleInfo, ReferringClinic, Patient, Doctor, SmiNetConfig, + LabDiagnosisType, LabResult, Laboratory, SmiNetLabExport, + SmiNetConfigNotFoundError, Notification, + SmiNetClient, SmiNetValidationError) """ @@ -15,13 +15,13 @@ try: config = SmiNetConfig.create_from_search_paths() -except ConfigNotFoundError: +except SmiNetConfigNotFoundError: pytest.xfail("This test requires a sminet_client configuration file") def generate_valid_contract_with_random_sample_id(): """ - Returns a tuple of the parameters required to upload sample info via client.create + Returns a tuple of the parameters required to upload sample info via client.create """ constant_date = datetime(2020, 4, 30) @@ -34,14 +34,15 @@ def generate_valid_contract_with_random_sample_id(): sample_info = SampleInfo(status=1, sample_id=sample_id, sample_date_arrival=constant_date, sample_date_referral=constant_date, sample_material="Svalg", sample_free_text_referral="Extra info") - referring_clinic = ReferringClinic("Clinic name", "", "C", Doctor("Some doctor")) + referring_clinic = ReferringClinic( + "Clinic name", "", "C", Doctor("Some doctor")) patient = Patient("121212-1212", "k", "Tolvan Tolvansson", 23) reporting_doctor = Doctor("Reporting Doctor") lab_diagnosis = LabDiagnosisType("SCOV2", "SARS-CoV-2 (covid-19)") lab_result = LabResult("C", lab_diagnosis) - notification = NotificationType(sample_info, reporting_doctor, referring_clinic, patient, - lab_result) + notification = Notification(sample_info, reporting_doctor, referring_clinic, patient, + lab_result) laboratory = Laboratory(config.lab_number, config.lab_name) export = SmiNetLabExport(request_created, laboratory, notification) return export @@ -54,23 +55,25 @@ def test_can_create_request(): client = SmiNetClient(config) export = generate_valid_contract_with_random_sample_id() timestamp = datetime.now().strftime("%y%m%dT%H%M%S") - file_name = "{}-{}".format(timestamp, export.notification.sample_info.sample_id) + file_name = "{}-{}".format(timestamp, + export.notification.sample_info.sample_id) client.create(export, file_name) def test_can_create_request_with_missing_info(): """ - Ensure that we can create info were the following data is missing + Ensure that we can create info were the following data is missing """ client = SmiNetClient(config) export = generate_valid_contract_with_random_sample_id() - export.notification.patient.patient_age = "1" # This can not be empty - export.notification.patient.patient_age = None + export.notification.patient.patient_age = "1" # This can not be empty + export.notification.patient.patient_age = None export.notification.patient.patient_name = "" timestamp = datetime.now().strftime("%y%m%dT%H%M%S") - file_name = "some-empty-{}-{}".format(timestamp, export.notification.sample_info.sample_id) + file_name = "some-empty-{}-{}".format(timestamp, + export.notification.sample_info.sample_id) client.create(export, file_name) @@ -92,5 +95,3 @@ def test_covid_request_xml_is_not_valid_against_xsd_schema(): xml.append(element) with pytest.raises(SmiNetValidationError): client.validate(xml) - - diff --git a/sminet-client/tests/unit/test_sminet_client.py b/sminet-client/tests/unit/test_sminet_client.py index 76cb53e..3d4686b 100644 --- a/sminet-client/tests/unit/test_sminet_client.py +++ b/sminet-client/tests/unit/test_sminet_client.py @@ -1,8 +1,6 @@ import os -from lxml import etree from datetime import datetime -from sminet_client import (SampleInfo, ReferringClinic, Patient, Doctor, SmiNetConfig, NotificationType, - LabDiagnosisType, LabResult, Laboratory, SmiNetLabExport) +from sminet_client import Doctor, Notification, LabDiagnosisType, LabResult, Laboratory, SmiNetLabExport, StatusType def get_fixture(fname): @@ -15,24 +13,28 @@ def create_document(): constant_date = datetime(2020, 4, 30) request_created = datetime(2020, 5, 18, 18, 11, 6) - sample_info = SampleInfo(status=1, sample_id="123", sample_date_arrival=constant_date, - sample_date_referral=constant_date, sample_material="Svalg", - sample_free_text_referral="Free text") - referring_clinic = ReferringClinic("Clinic name", "", "C", Doctor("Referring doctor")) - patient = Patient("1234", "k", "Patient Name", 23) - reporting_doctor = Doctor("Reporting Doctor") + sample_info = Notification.SampleInfo( + status=StatusType.FINAL_RESPONSE, + sample_id="123", + sample_date_arrival=constant_date, + sample_date_referral=constant_date, + sample_material="Svalg", + sample_free_text_referral="Free text") + referring_clinic = Notification.ReferringClinic( + "Clinic name", "", "C", Doctor("Referring doctor")) + patient = Notification.Patient("1234", "k", "Patient Name", 23) + reporting_doctor = Notification.Doctor("Reporting Doctor") lab_diagnosis = LabDiagnosisType("SCOV2", "SARS-CoV-2 (covid-19)") lab_result = LabResult("C", lab_diagnosis) - notification = NotificationType(sample_info, reporting_doctor, referring_clinic, patient, - lab_result) + notification = Notification(sample_info, reporting_doctor, referring_clinic, patient, + lab_result) laboratory = Laboratory(1, "Some lab") export = SmiNetLabExport(request_created, laboratory, notification) return export.to_document("http://stage.sminet.se/xml-schemas/SmiNetLabExport.xsd") + def test_can_create_expected_request(): expected = get_fixture("valid_request.xml") actual = create_document() - assert expected == actual - - + assert expected == actual From 94c2e1c3173be9d6b8654a89821b541d8c6e82b7 Mon Sep 17 00:00:00 2001 From: Steinar Sturlaugsson Date: Mon, 1 Jun 2020 15:24:33 +0200 Subject: [PATCH 07/14] Minor refactoring (naming and constants) --- clarity-ext-scripts/tests/integration/test_knm_sminet.py | 3 ++- sminet-client/sminet_client/__init__.py | 4 ++-- sminet-client/tests/integration/test_sminet_client.py | 9 ++++++--- sminet-client/tests/unit/test_sminet_client.py | 5 +++-- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/clarity-ext-scripts/tests/integration/test_knm_sminet.py b/clarity-ext-scripts/tests/integration/test_knm_sminet.py index 55c04b6..0f1850d 100644 --- a/clarity-ext-scripts/tests/integration/test_knm_sminet.py +++ b/clarity-ext-scripts/tests/integration/test_knm_sminet.py @@ -5,6 +5,7 @@ from clarity_ext_scripts.covid.partner_api_client import KARLSSON_AND_NOVAK, ORG_URI_BY_NAME from clarity_ext.cli import load_config from clarity_ext_scripts.covid.services.knm_service import KNMSampleAccessor, KNMService +from sminet_client import SampleMaterial """ @@ -31,7 +32,7 @@ def test_can_create_request(): integration = KNMSmiNetIntegrationService( config, knm_service, sminet_service) org_uri = ORG_URI_BY_NAME[KARLSSON_AND_NOVAK] - sample = KNMSampleAccessor(org_uri, "5236417647", constant_date, "Svalg") + sample = KNMSampleAccessor(org_uri, "5236417647", constant_date, SampleMaterial.SVALG) sample_free_text = "Anamnes: Personalprov" lab_result = SmiNetService.create_scov2_positive_lab_result() diff --git a/sminet-client/sminet_client/__init__.py b/sminet-client/sminet_client/__init__.py index 5f11f77..f0a2f3b 100644 --- a/sminet-client/sminet_client/__init__.py +++ b/sminet-client/sminet_client/__init__.py @@ -263,7 +263,7 @@ def __init__(self, status): self.value = status -class SampleMaterialType(object): +class SampleMaterial(object): """ Avser undersökningsmatrial. (Notera att det är tillåtet att byta ut 'å' och 'ä' mot 'a' samt att 'ö' kan bytas ut mot 'o'. Detta alternativ finns om det skulle bli problem med @@ -423,7 +423,7 @@ def __init__(self, # Is a string according to the docs self.sample_id = ShortString(str(sample_id)) self.sample_date_arrival = SmiNetDate(sample_date_arrival) - self.sample_material = SampleMaterialType(sample_material) + self.sample_material = SampleMaterial(sample_material) self.optional_reference = ShortLimitedString( optional_reference) if optional_reference else None self.sample_date_referral = SmiNetDate( diff --git a/sminet-client/tests/integration/test_sminet_client.py b/sminet-client/tests/integration/test_sminet_client.py index 9b4539f..6294193 100644 --- a/sminet-client/tests/integration/test_sminet_client.py +++ b/sminet-client/tests/integration/test_sminet_client.py @@ -5,7 +5,7 @@ import pytest from sminet_client import (SampleInfo, ReferringClinic, Patient, Doctor, SmiNetConfig, LabDiagnosisType, LabResult, Laboratory, SmiNetLabExport, - SmiNetConfigNotFoundError, Notification, + SmiNetConfigNotFoundError, Notification, SampleMaterial, SmiNetClient, SmiNetValidationError) @@ -31,8 +31,11 @@ def generate_valid_contract_with_random_sample_id(): for _ in range(25 - len(prefix))) sample_id = prefix + rnd - sample_info = SampleInfo(status=1, sample_id=sample_id, sample_date_arrival=constant_date, - sample_date_referral=constant_date, sample_material="Svalg", + sample_info = SampleInfo(status=1, + sample_id=sample_id, + sample_date_arrival=constant_date, + sample_date_referral=constant_date, + sample_material=SampleMaterial.SVALG, sample_free_text_referral="Extra info") referring_clinic = ReferringClinic( "Clinic name", "", "C", Doctor("Some doctor")) diff --git a/sminet-client/tests/unit/test_sminet_client.py b/sminet-client/tests/unit/test_sminet_client.py index 3d4686b..148d630 100644 --- a/sminet-client/tests/unit/test_sminet_client.py +++ b/sminet-client/tests/unit/test_sminet_client.py @@ -1,6 +1,7 @@ import os from datetime import datetime -from sminet_client import Doctor, Notification, LabDiagnosisType, LabResult, Laboratory, SmiNetLabExport, StatusType +from sminet_client import (Doctor, Notification, LabDiagnosisType, + LabResult, Laboratory, SmiNetLabExport, StatusType, SampleMaterial) def get_fixture(fname): @@ -18,7 +19,7 @@ def create_document(): sample_id="123", sample_date_arrival=constant_date, sample_date_referral=constant_date, - sample_material="Svalg", + sample_material=SampleMaterial.SVALG, sample_free_text_referral="Free text") referring_clinic = Notification.ReferringClinic( "Clinic name", "", "C", Doctor("Referring doctor")) From a0a596b0f819c5d535711468b973b2e460f47df5 Mon Sep 17 00:00:00 2001 From: Steinar Sturlaugsson Date: Mon, 1 Jun 2020 15:35:15 +0200 Subject: [PATCH 08/14] Debug .genologicsrc issue --- .github/workflows/pythonapp.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 51b0ceb..240c97b 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -21,7 +21,8 @@ jobs: python-version: 2.7 - name: Add dummy genologicsrc run: | - echo "[genologics]\nBASEURI=https://url\nUSERNAME=username\nPASSWORD=pass\n" > ~/.genologicsrc + echo "[genologics]\nBASEURI=https://fancy.server\nUSERNAME=username\nPASSWORD=pass\n" > ~/.genologicsrc + cat ~/.genologicsrc - name: Install dependencies run: | ./clarity-ext-scripts/setup.sh From 39091a8f51e72b08fad07e351495cecc5bb4e6f8 Mon Sep 17 00:00:00 2001 From: Steinar Sturlaugsson Date: Mon, 1 Jun 2020 15:40:14 +0200 Subject: [PATCH 09/14] Use heredoc in github action --- .github/workflows/pythonapp.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 240c97b..2644c65 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -21,7 +21,12 @@ jobs: python-version: 2.7 - name: Add dummy genologicsrc run: | - echo "[genologics]\nBASEURI=https://fancy.server\nUSERNAME=username\nPASSWORD=pass\n" > ~/.genologicsrc + cat << EOF > ~/.genologicsrc +[genologics] +BASEURI=https://fancy.server +USERNAME=username +PASSWORD=pass +EOF cat ~/.genologicsrc - name: Install dependencies run: | From e964ea8d0bf69e5717e7a2653305ad978f1f8a24 Mon Sep 17 00:00:00 2001 From: Steinar Sturlaugsson Date: Mon, 1 Jun 2020 15:42:46 +0200 Subject: [PATCH 10/14] Fix yaml syntax --- .github/workflows/pythonapp.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 2644c65..9dd58d7 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -22,11 +22,11 @@ jobs: - name: Add dummy genologicsrc run: | cat << EOF > ~/.genologicsrc -[genologics] -BASEURI=https://fancy.server -USERNAME=username -PASSWORD=pass -EOF + [genologics] + BASEURI=https://fancy.server + USERNAME=username + PASSWORD=pass + EOF cat ~/.genologicsrc - name: Install dependencies run: | From 85fcfbe33787650b20a053aeabd43d56d0c8ccf2 Mon Sep 17 00:00:00 2001 From: Steinar Sturlaugsson Date: Mon, 1 Jun 2020 15:49:40 +0200 Subject: [PATCH 11/14] Move unit tests and ignore integration on build server --- .github/workflows/pythonapp.yml | 4 ++-- .../tests/integration/test_knm_sminet.py | 6 +----- .../tests/{ => unit}/test_label_printer.py | 0 .../tests/{ => unit}/test_partner_client.py | 0 .../tests/{ => unit}/test_rtpcr_analysis_service.py | 0 sminet-client/tests/integration/test_sminet_client.py | 10 +++------- 6 files changed, 6 insertions(+), 14 deletions(-) rename clarity-ext-scripts/tests/{ => unit}/test_label_printer.py (100%) rename clarity-ext-scripts/tests/{ => unit}/test_partner_client.py (100%) rename clarity-ext-scripts/tests/{ => unit}/test_rtpcr_analysis_service.py (100%) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 9dd58d7..0452373 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -34,6 +34,6 @@ jobs: - name: Test with pytest run: | pip install pytest - pytest ./clarity-ext-scripts/tests - pytest ./sminet-client/tests + pytest ./clarity-ext-scripts/tests/unit + pytest ./sminet-client/tests/unit diff --git a/clarity-ext-scripts/tests/integration/test_knm_sminet.py b/clarity-ext-scripts/tests/integration/test_knm_sminet.py index 0f1850d..df54dd0 100644 --- a/clarity-ext-scripts/tests/integration/test_knm_sminet.py +++ b/clarity-ext-scripts/tests/integration/test_knm_sminet.py @@ -1,4 +1,3 @@ -import pytest from datetime import datetime from clarity_ext_scripts.covid.services.sminet_service import SmiNetService from clarity_ext_scripts.covid.services.knm_sminet_service import KNMSmiNetIntegrationService @@ -12,10 +11,7 @@ Tests the SmiNet/KNM integration """ -try: - config = load_config() -except IOError: - pytest.xfail("This test requires a sminet_client configuration file") +config = load_config() def test_can_create_request(): diff --git a/clarity-ext-scripts/tests/test_label_printer.py b/clarity-ext-scripts/tests/unit/test_label_printer.py similarity index 100% rename from clarity-ext-scripts/tests/test_label_printer.py rename to clarity-ext-scripts/tests/unit/test_label_printer.py diff --git a/clarity-ext-scripts/tests/test_partner_client.py b/clarity-ext-scripts/tests/unit/test_partner_client.py similarity index 100% rename from clarity-ext-scripts/tests/test_partner_client.py rename to clarity-ext-scripts/tests/unit/test_partner_client.py diff --git a/clarity-ext-scripts/tests/test_rtpcr_analysis_service.py b/clarity-ext-scripts/tests/unit/test_rtpcr_analysis_service.py similarity index 100% rename from clarity-ext-scripts/tests/test_rtpcr_analysis_service.py rename to clarity-ext-scripts/tests/unit/test_rtpcr_analysis_service.py diff --git a/sminet-client/tests/integration/test_sminet_client.py b/sminet-client/tests/integration/test_sminet_client.py index 6294193..d6c2b67 100644 --- a/sminet-client/tests/integration/test_sminet_client.py +++ b/sminet-client/tests/integration/test_sminet_client.py @@ -5,18 +5,14 @@ import pytest from sminet_client import (SampleInfo, ReferringClinic, Patient, Doctor, SmiNetConfig, LabDiagnosisType, LabResult, Laboratory, SmiNetLabExport, - SmiNetConfigNotFoundError, Notification, SampleMaterial, - SmiNetClient, SmiNetValidationError) + Notification, SampleMaterial, SmiNetClient, SmiNetValidationError) """ Tests an integration to the SmiNet. Requires a configuration file (see README.md) """ -try: - config = SmiNetConfig.create_from_search_paths() -except SmiNetConfigNotFoundError: - pytest.xfail("This test requires a sminet_client configuration file") +config = SmiNetConfig.create_from_search_paths() def generate_valid_contract_with_random_sample_id(): @@ -53,7 +49,7 @@ def generate_valid_contract_with_random_sample_id(): def test_can_create_request(): """ - Tests creating a request. The test is xfailed if the configuration is not available + Tests creating a request. """ client = SmiNetClient(config) export = generate_valid_contract_with_random_sample_id() From 5eb92266c01002c9ff7f45e15ba5c56f4c3eb379 Mon Sep 17 00:00:00 2001 From: Steinar Sturlaugsson Date: Mon, 1 Jun 2020 19:19:22 +0200 Subject: [PATCH 12/14] Fix namespaces --- .../covid/create_samples/assign_unregistered_to_anonymous.py | 2 +- .../clarity_ext_scripts/covid/report_results.py | 2 +- .../clarity_ext_scripts/covid/validate_sample_creation_list.py | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/create_samples/assign_unregistered_to_anonymous.py b/clarity-ext-scripts/clarity_ext_scripts/covid/create_samples/assign_unregistered_to_anonymous.py index 2091b85..9ae8d40 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/create_samples/assign_unregistered_to_anonymous.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/create_samples/assign_unregistered_to_anonymous.py @@ -1,7 +1,7 @@ from uuid import uuid4 from clarity_ext.extensions import GeneralExtension from clarity_ext_scripts.covid.create_samples.common import ValidatedSampleListFile -from clarity_ext_scripts.covid.knm_service import KNMClientFromExtension +from clarity_ext_scripts.covid.services.knm_service import KNMClientFromExtension from clarity_ext_scripts.covid.partner_api_client import ( ORG_URI_BY_NAME, TESTING_ORG, CouldNotCreateServiceRequest, ServiceRequestAlreadyExists) diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/report_results.py b/clarity-ext-scripts/clarity_ext_scripts/covid/report_results.py index 20ee968..5614204 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/report_results.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/report_results.py @@ -6,7 +6,7 @@ PartnerClientAPIException) from clarity_ext_scripts.covid.rtpcr_analysis_service import FAILED_STATES from clarity_ext_scripts.covid.controls import Controls -from clarity_ext_scripts.covid.knm_service import KNMClientFromExtension +from clarity_ext_scripts.covid.services.knm_service import KNMClientFromExtension logger = logging.getLogger(__name__) diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/validate_sample_creation_list.py b/clarity-ext-scripts/clarity_ext_scripts/covid/validate_sample_creation_list.py index bc099bd..1a455e4 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/validate_sample_creation_list.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/validate_sample_creation_list.py @@ -1,9 +1,8 @@ import cStringIO import logging from datetime import datetime -from clarity_ext.extensions import GeneralExtension from clarity_ext_scripts.covid.controls import controls_barcode_generator -from clarity_ext_scripts.covid.knm_service import KNMClientFromExtension +from clarity_ext_scripts.covid.services.knm_service import KNMClientFromExtension from clarity_ext_scripts.covid.create_samples.common import ( BaseRawSampleListFile, ValidatedSampleListFile, BaseValidateRawSampleListExtension, BUTTON_TEXT_ASSIGN_UNREGISTERED_TO_ANONYMOUS From 449bd2693dda7e99b8e6c31a7977ae9afc7a966f Mon Sep 17 00:00:00 2001 From: Steinar Sturlaugsson Date: Mon, 1 Jun 2020 19:50:11 +0200 Subject: [PATCH 13/14] Extension for the KNM/SmiNet integration --- .../covid/report_to_sminet.py | 132 ++++++++++++++++++ .../covid/services/knm_sminet_service.py | 19 ++- .../covid/services/sminet_service.py | 10 ++ .../clarity_ext_scripts/covid/utils.py | 36 ++++- clarity-ext-scripts/repos/clarity-ext | 2 +- sminet-client/sminet_client/__init__.py | 8 +- 6 files changed, 192 insertions(+), 15 deletions(-) create mode 100644 clarity-ext-scripts/clarity_ext_scripts/covid/report_to_sminet.py diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/report_to_sminet.py b/clarity-ext-scripts/clarity_ext_scripts/covid/report_to_sminet.py new file mode 100644 index 0000000..fda72f9 --- /dev/null +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/report_to_sminet.py @@ -0,0 +1,132 @@ +from datetime import datetime +import logging +from sminet_client import SampleMaterial, SmiNetError +from clarity_ext.extensions import GeneralExtension +from clarity_ext_scripts.covid.utils import CtmrCovidSubstanceInfo +from clarity_ext_scripts.covid.services.sminet_service import SmiNetService +from clarity_ext_scripts.covid.services.knm_service import KNMSampleAccessor +from clarity_ext_scripts.covid.services.knm_sminet_service import ( + KNMSmiNetIntegrationService, IntegrationError, UnregisteredPatient) +from clarity_ext_scripts.covid.partner_api_client import ( + COVID_RESPONSE_POSITIVE, PartnerClientAPIException) + + +UDF_TRUE = "Yes" +logger = logging.getLogger(__name__) + + +def should_report(substance): + """ + A substance is ignored from SmiNet reporting if it's a control, has been run before or doesn't + have a positive result. It is also ignored if the research engineer has set the UDF + "SmiNet status" to "ignore" + + Samples that should be reported must be in the SmiNet state `retry` or `error` or None + """ + if substance.is_control: + return False + + if substance.submitted_sample.udf_rtpcr_covid19_result_latest != COVID_RESPONSE_POSITIVE: + return False + + if substance.sminet_status in [SmiNetService.STATUS_IGNORE, SmiNetService.STATUS_SUCCESS]: + return False + + if substance.sminet_status not in [ + None, + SmiNetService.STATUS_RETRY, + SmiNetService.STATUS_ERROR]: + raise AssertionError( + "Unexpected SmiNet status: {}".format(substance.sminet_status)) + + return True + + +class Extension(GeneralExtension): + """ + Reports results to SmiNet. + + For all analytes in the step: + * If sminet_status is not set: + * Checks if it should be imported. + * If it shouldn't, updates it to "ignore" + * If it should, tries to send the data + * If sminet_status is "error", retries it + * If sminet_status is "ignore", ignores it + + + Test data required: + * Positive => success + * Negative => ignore + * Failed => ignore + * Anonymous => ignore + """ + + def report(self, substance): + """ + Reports this substance to SmiNet and updates the status in the LIMS. + """ + + org_referral_code = substance.submitted_sample.name.split("_")[0] + date_arrival = substance.submitted_sample.api_resource.date_received + date_arrival = datetime.strptime(date_arrival, "%Y-%m-%d") + + sample = KNMSampleAccessor(substance.submitted_sample.udf_knm_org_uri, + org_referral_code, + date_arrival, + SampleMaterial.SVALG) + + integration = KNMSmiNetIntegrationService(self.config) + lab_result = SmiNetService.create_scov2_positive_lab_result() + + error_msg = "" + + # TODO: Anonymous should be ignored + try: + integration.export_to_sminet(sample, + doctor_name="Lars Engstrand", + lab_result=lab_result, + sample_free_text="Anamnes: Personalprov") + status = SmiNetService.STATUS_SUCCESS + except UnregisteredPatient: + status = SmiNetService.STATUS_IGNORE + except PartnerClientAPIException as knm_error: + error_msg = knm_error.message + self.usage_error_defer("Error while fetching data from KNM", + substance.submitted_sample.name) + status = SmiNetService.STATUS_ERROR + except SmiNetError as sminet_error: + error_msg = sminet_error.message + self.usage_error_defer("Error while uploading sample to SmiNet", + substance.submitted_sample.name) + status = SmiNetService.STATUS_ERROR + except IntegrationError as int_error: + self.usage_error_defer("Error while uploading sample to SmiNet", + substance.submitted_sample.name) + error_msg = int_error.message + status = SmiNetService.STATUS_ERROR + + timestamp = datetime.now().strftime("%y%m%dT%H%M%S") + + substance.submitted_sample.udf_map.force("SmiNet status", status) + substance.substance.udf_map.force("SmiNet status", status) + + substance.submitted_sample.udf_map.force( + "SmiNet uploaded date", timestamp) + substance.submitted_sample.udf_map.force("SmiNet artifact source", + substance.substance.api_resource.uri) + substance.submitted_sample.udf_map.force( + "SmiNet last error", error_msg) + self.context.update(substance.submitted_sample) + self.context.update(substance.substance) + self.context.commit() + + def execute(self): + for plate in self.context.input_containers: + for well in plate.occupied: + substance = CtmrCovidSubstanceInfo(well.artifact) + if should_report(substance): + self.report(substance) + + def integration_tests(self): + yield "24-48808" diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/services/knm_sminet_service.py b/clarity-ext-scripts/clarity_ext_scripts/covid/services/knm_sminet_service.py index abd584a..d1bdf52 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/services/knm_sminet_service.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/services/knm_sminet_service.py @@ -3,7 +3,8 @@ from sminet_client import (SampleInfo, ReferringClinic, Patient, Doctor, SmiNetLabExport, StatusType, Notification) from sminet_client import Gender as SmiNetGender -from clarity_ext_scripts.covid.services.knm_service import ServiceRequestProvider +from clarity_ext_scripts.covid.services.knm_service import ServiceRequestProvider, KNMService +from clarity_ext_scripts.covid.services.sminet_service import SmiNetService class KNMSmiNetIntegrationService(object): @@ -16,10 +17,10 @@ class KNMSmiNetIntegrationService(object): None: SmiNetGender.UNKNOWN, } - def __init__(self, config, knm_service, sminet_service): + def __init__(self, config, knm_service=None, sminet_service=None): self.config = config - self.sminet_service = sminet_service - self.knm_service = knm_service + self.knm_service = knm_service or KNMService(config) + self.sminet_service = sminet_service or SmiNetService(config) def get_county_from_organization(self, organization): """ @@ -63,17 +64,17 @@ def patient_identifier(): try: patient_identifier = provider.patient["identifier"] except KeyError: - raise IntegrationError( + raise UnregisteredPatient( "Missing field 'identifier' on the patient resource for {}".format(provider)) if len(patient_identifier) == 0: - raise IntegrationError( + raise UnregisteredPatient( "Field 'identifier' is empty on patient resource for {}".format(provider)) try: return patient_identifier[0]["value"] except KeyError: - raise IntegrationError( + raise UnregisteredPatient( "First entry in 'identifier' doesn't have a value key for {}".format(provider)) def patient_gender(): @@ -135,3 +136,7 @@ def export_to_sminet(self, sample, doctor_name, lab_result, sample_free_text): class IntegrationError(Exception): pass + + +class UnregisteredPatient(IntegrationError): + pass diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/services/sminet_service.py b/clarity-ext-scripts/clarity_ext_scripts/covid/services/sminet_service.py index d5d4ecb..736145b 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/services/sminet_service.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/services/sminet_service.py @@ -4,6 +4,16 @@ class SmiNetService(object): + STATUS_SUCCESS = "success" # We've already uploaded the results to SmiNet + STATUS_ERROR = "error" # We've tried to upload the results to SmiNet but failed + + # The sample shouldn't be reported even if it's in the report step. This can be either + # because the extension figures this out based on business rules, or a research engineer + # marks it manually as such + STATUS_IGNORE = "ignore" + + STATUS_RETRY = "retry" + def __init__(self, config): self.config = SmiNetConfig(**config) self.client = SmiNetClient(self.config) diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/utils.py b/clarity-ext-scripts/clarity_ext_scripts/covid/utils.py index 56b31a8..d336dd6 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/utils.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/utils.py @@ -1,7 +1,7 @@ import re import time from datetime import datetime -from clarity_ext_scripts.covid.partner_api_client import PartnerAPIV7Client +from clarity_ext.domain import Sample, Artifact class UniqueBarcodeGenerator(object): @@ -9,7 +9,7 @@ class UniqueBarcodeGenerator(object): Generates a unique barcode ID that's guaranteed to be unique if there is only one person generating barcodes at a time (they use the current second to mark a unique time), while also being short enough to fit within a limited length barcode (currently 14 characters). - + Format: <2 digit type_id (0-99)><8 chars timestamp in hex><3 digits running> """ @@ -21,9 +21,9 @@ def __init__(self, prefix): self.pattern = re.compile( "^" + prefix + - "(?P\d{2})" + - "(?P\w{8})" + - "(?P\d{3})" + + r"(?P\d{2})" + + r"(?P\w{8})" + + r"(?P\d{3})" + "$") def parse(self, barcode): @@ -105,6 +105,13 @@ def _deduce_control_type_from_sample(sample): except AttributeError: return None + @property + def submitted_sample(self): + if isinstance(self.substance, Sample): + return self.substance + elif isinstance(self.substance, Artifact): + return self.substance.sample() + @staticmethod def _deduce_control_type_from_analyte_name(name): """ @@ -133,3 +140,22 @@ def control_type(self): else: raise NotImplementedError("Not implemented substance type {}".format( type(self.substance))) + + @property + def is_control(self): + return self.control_type is not None + + @property + def sminet_status(self): + try: + return self.substance.udf_sminet_status + except AttributeError: + return None + + @sminet_status.setter + def sminet_status(self, value): + self.submitted_sample.udf_sminet_status = value + self.substance.udf_sminet_status = value + + def __str__(self): + return "name={}, is_control={}".format(self.substance.name, self.is_control) diff --git a/clarity-ext-scripts/repos/clarity-ext b/clarity-ext-scripts/repos/clarity-ext index b3f6639..ad6cf90 160000 --- a/clarity-ext-scripts/repos/clarity-ext +++ b/clarity-ext-scripts/repos/clarity-ext @@ -1 +1 @@ -Subproject commit b3f66395e3bfbed3f712f1160b767cf750514781 +Subproject commit ad6cf90137aa1e1a5a71a56b37d9c275f9432c20 diff --git a/sminet-client/sminet_client/__init__.py b/sminet-client/sminet_client/__init__.py index f0a2f3b..a3f8f1d 100644 --- a/sminet-client/sminet_client/__init__.py +++ b/sminet-client/sminet_client/__init__.py @@ -606,11 +606,15 @@ def to_document(self, xsd_url): return ET.tostring(export, pretty_print=True, encoding="ISO-8859-1") -class SmiNetValidationError(Exception): +class SmiNetError(Exception): pass -class SmiNetRequestError(Exception): +class SmiNetValidationError(SmiNetError): + pass + + +class SmiNetRequestError(SmiNetError): pass From 4b8f06640f322c2120bb89eb764275df33cf474e Mon Sep 17 00:00:00 2001 From: Steinar Sturlaugsson Date: Tue, 2 Jun 2020 09:22:47 +0200 Subject: [PATCH 14/14] Remove TODO --- .../clarity_ext_scripts/covid/report_to_sminet.py | 1 - 1 file changed, 1 deletion(-) diff --git a/clarity-ext-scripts/clarity_ext_scripts/covid/report_to_sminet.py b/clarity-ext-scripts/clarity_ext_scripts/covid/report_to_sminet.py index fda72f9..7d92169 100644 --- a/clarity-ext-scripts/clarity_ext_scripts/covid/report_to_sminet.py +++ b/clarity-ext-scripts/clarity_ext_scripts/covid/report_to_sminet.py @@ -81,7 +81,6 @@ def report(self, substance): error_msg = "" - # TODO: Anonymous should be ignored try: integration.export_to_sminet(sample, doctor_name="Lars Engstrand",