diff --git a/.github/workflows/dev-release.yaml b/.github/workflows/dev-release.yaml index b1b8e24..561ff32 100644 --- a/.github/workflows/dev-release.yaml +++ b/.github/workflows/dev-release.yaml @@ -15,6 +15,7 @@ jobs: architecture: 'x64' - uses: actions/checkout@v2 with: + ref: refs/pull/${{github.event.issue.number}}/merge fetch-depth: 0 - name: Install Poetry and dependencies uses: SneaksAndData/github-actions/install_poetry@v0.0.7 diff --git a/esd_services_api_client/crystal/__init__.py b/esd_services_api_client/crystal/__init__.py index 603e652..310ec91 100644 --- a/esd_services_api_client/crystal/__init__.py +++ b/esd_services_api_client/crystal/__init__.py @@ -1,4 +1,4 @@ """init file""" -from esd_services_api_client.crystal._connector import CrystalConnector, add_crystal_args, extract_crystal_args -from esd_services_api_client.crystal._models import AlgorithmRunResult, RequestLifeCycleStage, RequestResult, CrystalEntrypointArguments +from esd_services_api_client.crystal._connector import * +from esd_services_api_client.crystal._models import * diff --git a/esd_services_api_client/crystal/_connector.py b/esd_services_api_client/crystal/_connector.py index 36a82db..7c26ee5 100644 --- a/esd_services_api_client/crystal/_connector.py +++ b/esd_services_api_client/crystal/_connector.py @@ -3,13 +3,14 @@ """ import os from argparse import Namespace, ArgumentParser -from typing import Dict, Optional, Type, TypeVar +from typing import Dict, Optional, Type, TypeVar, List from requests.auth import HTTPBasicAuth from proteus.utils import session_with_retries from proteus.storage.models.format import SerializationFormat -from esd_services_api_client.crystal._models import RequestResult, AlgorithmRunResult, CrystalEntrypointArguments +from esd_services_api_client.crystal._models import RequestResult, AlgorithmRunResult, CrystalEntrypointArguments, \ + AlgorithmRequest, AlgorithmConfiguration T = TypeVar('T') # pylint: disable=C0103 @@ -87,19 +88,24 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): self.dispose() - def create_run(self, algorithm: str, payload: Dict, api_version: str = "v1.1") -> str: + def create_run(self, algorithm: str, payload: Dict, api_version: str = "v1.1", + custom_config: Optional[AlgorithmConfiguration] = None, tag: Optional[str] = None) -> str: """ Creates a Crystal job run against the latest API version. :param algorithm: Name of a connected algorithm. :param payload: Algorithm payload. :param api_version: Crystal API version. + :param custom_config: Customized config for this run. + :param tag: Client-side submission identifier. :return: Request identifier assigned to the job by Crystal. """ - run_body = { - "AlgorithmName": algorithm, - "AlgorithmParameters": payload - } + run_body = AlgorithmRequest( + algorithm_name=algorithm, + algorithm_parameters=payload, + custom_configuration=custom_config, + tag=tag + ).to_dict() print(f"Sending the following configuration for algorithm {algorithm}: {run_body}") @@ -133,6 +139,22 @@ def retrieve_run(self, run_id: str, api_version: str = "v1.1") -> RequestResult: return crystal_result + def retrieve_runs(self, tag: str, api_version: str = "v1.1") -> List[RequestResult]: + """ + Retrieves all submitted Crystal jobs with matching tags. + + :param tag: A request tag assigned by a client. + :param api_version: Crystal API version. + """ + url = f'{self.base_url}/algorithm/{api_version}/tag/{tag}/results' + + response = self.http.get(url=url) + + # raise if not successful + response.raise_for_status() + + return [RequestResult.from_dict(run_result) for run_result in response.json()] + def submit_result(self, result: AlgorithmRunResult, url: str) -> None: """ Submit a result of an algorithm back to Crystal. diff --git a/esd_services_api_client/crystal/_models.py b/esd_services_api_client/crystal/_models.py index a712048..ea4e3f8 100644 --- a/esd_services_api_client/crystal/_models.py +++ b/esd_services_api_client/crystal/_models.py @@ -2,8 +2,10 @@ Models for Crystal connector """ from enum import Enum -from dataclasses import dataclass -from typing import Optional +from dataclasses import dataclass, field +from typing import Optional, Dict, List + +from dataclasses_json import dataclass_json, LetterCase, DataClassJsonMixin, config class RequestLifeCycleStage(Enum): @@ -20,32 +22,21 @@ class RequestLifeCycleStage(Enum): THROTTLED = 'THROTTLED' +@dataclass_json(letter_case=LetterCase.CAMEL) @dataclass -class RequestResult: +class RequestResult(DataClassJsonMixin): """ The Crystal result when retrieving an existing run. """ - run_id: str - status: RequestLifeCycleStage + run_id: str = field(metadata=config(field_name='requestId')) + status: RequestLifeCycleStage = field( + metadata=config( + encoder=lambda v: v.value if v else None, + decoder=RequestLifeCycleStage + ), default=None) result_uri: Optional[str] = None run_error_message: Optional[str] = None - @classmethod - def from_dict(cls, dict_: dict): - """ - Constructs a CrystalResult object from a dictionary containing the - keys from the /result HTTP GET request. - - :param dict_: The (JSON) dict from the HTTP request. - :return: The corresponding CrystalResult object. - """ - return RequestResult( - run_id=dict_['requestId'], - status=RequestLifeCycleStage(dict_['status']), - result_uri=dict_['resultUri'], - run_error_message=dict_['runErrorMessage'], - ) - @dataclass class AlgorithmRunResult: @@ -67,3 +58,61 @@ class CrystalEntrypointArguments: request_id: str results_receiver: str sign_result: Optional[bool] = None + + +class AlgorithmConfigurationValueType(Enum): + """ + Value type for algorithm config maps and secrets. + + PLAIN - plain text value + RELATIVE_REFERENCE - reference to a file deployed alongside algorithm config. + """ + PLAIN = "PLAIN" + RELATIVE_REFERENCE = "RELATIVE_REFERENCE" + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class AlgorithmConfigurationEntry(DataClassJsonMixin): + """ + Crystal algorithm configuration entry. + """ + name: str + value: str + value_type: Optional[AlgorithmConfigurationValueType] = field( + metadata=config( + encoder=lambda v: v.value if v else None, + decoder=AlgorithmConfigurationValueType + ), default=None) + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class AlgorithmConfiguration(DataClassJsonMixin): + """ + Crystal algorithm configuration. Used for overriding defaults. + """ + image_repository: Optional[str] = None + image_tag: Optional[str] = None + deadline_seconds: Optional[int] = None + maximum_retries: Optional[int] = None + env: Optional[List[AlgorithmConfigurationEntry]] = None + secrets: Optional[List[str]] = None + args: Optional[List[AlgorithmConfigurationEntry]] = None + cpu_limit: Optional[str] = None + memory_limit: Optional[str] = None + workgroup: Optional[str] = None + version: Optional[str] = None + monitoring_parameters: Optional[List[str]] = None + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class AlgorithmRequest(DataClassJsonMixin): + """ + Crystal algorthm request. + """ + algorithm_name: str + algorithm_parameters: Dict + custom_configuration: Optional[AlgorithmConfiguration] = None + tag: Optional[str] = None