Add tag support for crystal (#11)

.github/workflows/dev-release.yaml

@@ -15,6 +15,7 @@ jobs:
           architecture: 'x64'
       - uses: actions/checkout@v2
         with:
+          ref: refs/pull/${{github.event.issue.number}}/merge
           fetch-depth: 0
       - name: Install Poetry and dependencies
         uses: SneaksAndData/github-actions/[email protected]

esd_services_api_client/crystal/__init__.py

@@ -1,4 +1,4 @@
 """init file"""
 
-from esd_services_api_client.crystal._connector import CrystalConnector, add_crystal_args, extract_crystal_args
-from esd_services_api_client.crystal._models import AlgorithmRunResult, RequestLifeCycleStage, RequestResult, CrystalEntrypointArguments
+from esd_services_api_client.crystal._connector import *
+from esd_services_api_client.crystal._models import *

esd_services_api_client/crystal/_connector.py

@@ -3,13 +3,14 @@
 """
 import os
 from argparse import Namespace, ArgumentParser
-from typing import Dict, Optional, Type, TypeVar
+from typing import Dict, Optional, Type, TypeVar, List
 
 from requests.auth import HTTPBasicAuth
 
 from proteus.utils import session_with_retries
 from proteus.storage.models.format import SerializationFormat
-from esd_services_api_client.crystal._models import RequestResult, AlgorithmRunResult, CrystalEntrypointArguments
+from esd_services_api_client.crystal._models import RequestResult, AlgorithmRunResult, CrystalEntrypointArguments, \
+    AlgorithmRequest, AlgorithmConfiguration
 
 T = TypeVar('T')  # pylint: disable=C0103
 
@@ -87,19 +88,24 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.dispose()
 
-    def create_run(self, algorithm: str, payload: Dict, api_version: str = "v1.1") -> str:
+    def create_run(self, algorithm: str, payload: Dict, api_version: str = "v1.1",
+                   custom_config: Optional[AlgorithmConfiguration] = None, tag: Optional[str] = None) -> str:
         """
           Creates a Crystal job run against the latest API version.
 
         :param algorithm: Name of a connected algorithm.
         :param payload: Algorithm payload.
         :param api_version: Crystal API version.
+        :param custom_config: Customized config for this run.
+        :param tag: Client-side submission identifier.
         :return: Request identifier assigned to the job by Crystal.
         """
-        run_body = {
-            "AlgorithmName": algorithm,
-            "AlgorithmParameters": payload
-        }
+        run_body = AlgorithmRequest(
+            algorithm_name=algorithm,
+            algorithm_parameters=payload,
+            custom_configuration=custom_config,
+            tag=tag
+        ).to_dict()
 
         print(f"Sending the following configuration for algorithm {algorithm}: {run_body}")
 
@@ -133,6 +139,22 @@ def retrieve_run(self, run_id: str, api_version: str = "v1.1") -> RequestResult:
 
         return crystal_result
 
+    def retrieve_runs(self, tag: str, api_version: str = "v1.1") -> List[RequestResult]:
+        """
+          Retrieves all submitted Crystal jobs with matching tags.
+
+          :param tag: A request tag assigned by a client.
+          :param api_version: Crystal API version.
+        """
+        url = f'{self.base_url}/algorithm/{api_version}/tag/{tag}/results'
+
+        response = self.http.get(url=url)
+
+        # raise if not successful
+        response.raise_for_status()
+
+        return [RequestResult.from_dict(run_result) for run_result in response.json()]
+
     def submit_result(self, result: AlgorithmRunResult, url: str) -> None:
         """
         Submit a result of an algorithm back to Crystal.

esd_services_api_client/crystal/_models.py

@@ -2,8 +2,10 @@
   Models for Crystal connector
 """
 from enum import Enum
-from dataclasses import dataclass
-from typing import Optional
+from dataclasses import dataclass, field
+from typing import Optional, Dict, List
+
+from dataclasses_json import dataclass_json, LetterCase, DataClassJsonMixin, config
 
 
 class RequestLifeCycleStage(Enum):
@@ -20,32 +22,21 @@ class RequestLifeCycleStage(Enum):
     THROTTLED = 'THROTTLED'
 
 
+@dataclass_json(letter_case=LetterCase.CAMEL)
 @dataclass
-class RequestResult:
+class RequestResult(DataClassJsonMixin):
     """
     The Crystal result when retrieving an existing run.
     """
-    run_id: str
-    status: RequestLifeCycleStage
+    run_id: str = field(metadata=config(field_name='requestId'))
+    status: RequestLifeCycleStage = field(
+        metadata=config(
+            encoder=lambda v: v.value if v else None,
+            decoder=RequestLifeCycleStage
+        ), default=None)
     result_uri: Optional[str] = None
     run_error_message: Optional[str] = None
 
-    @classmethod
-    def from_dict(cls, dict_: dict):
-        """
-        Constructs a CrystalResult object from a dictionary containing the
-        keys from the /result HTTP GET request.
-
-        :param dict_: The (JSON) dict from the HTTP request.
-        :return: The corresponding CrystalResult object.
-        """
-        return RequestResult(
-            run_id=dict_['requestId'],
-            status=RequestLifeCycleStage(dict_['status']),
-            result_uri=dict_['resultUri'],
-            run_error_message=dict_['runErrorMessage'],
-        )
-
 
 @dataclass
 class AlgorithmRunResult:
@@ -67,3 +58,61 @@ class CrystalEntrypointArguments:
     request_id: str
     results_receiver: str
     sign_result: Optional[bool] = None
+
+
+class AlgorithmConfigurationValueType(Enum):
+    """
+      Value type for algorithm config maps and secrets.
+
+      PLAIN - plain text value
+      RELATIVE_REFERENCE - reference to a file deployed alongside algorithm config.
+    """
+    PLAIN = "PLAIN"
+    RELATIVE_REFERENCE = "RELATIVE_REFERENCE"
+
+
+@dataclass_json(letter_case=LetterCase.CAMEL)
+@dataclass
+class AlgorithmConfigurationEntry(DataClassJsonMixin):
+    """
+      Crystal algorithm configuration entry.
+    """
+    name: str
+    value: str
+    value_type: Optional[AlgorithmConfigurationValueType] = field(
+        metadata=config(
+            encoder=lambda v: v.value if v else None,
+            decoder=AlgorithmConfigurationValueType
+        ), default=None)
+
+
+@dataclass_json(letter_case=LetterCase.CAMEL)
+@dataclass
+class AlgorithmConfiguration(DataClassJsonMixin):
+    """
+     Crystal algorithm configuration. Used for overriding defaults.
+    """
+    image_repository: Optional[str] = None
+    image_tag: Optional[str] = None
+    deadline_seconds: Optional[int] = None
+    maximum_retries: Optional[int] = None
+    env: Optional[List[AlgorithmConfigurationEntry]] = None
+    secrets: Optional[List[str]] = None
+    args: Optional[List[AlgorithmConfigurationEntry]] = None
+    cpu_limit: Optional[str] = None
+    memory_limit: Optional[str] = None
+    workgroup: Optional[str] = None
+    version: Optional[str] = None
+    monitoring_parameters: Optional[List[str]] = None
+
+
+@dataclass_json(letter_case=LetterCase.CAMEL)
+@dataclass
+class AlgorithmRequest(DataClassJsonMixin):
+    """
+      Crystal algorthm request.
+    """
+    algorithm_name: str
+    algorithm_parameters: Dict
+    custom_configuration: Optional[AlgorithmConfiguration] = None
+    tag: Optional[str] = None