From eb15a46a3594ab23ded0abc87a3c4a9a60689db0 Mon Sep 17 00:00:00 2001 From: George Zubrienko Date: Tue, 7 Nov 2023 13:48:55 +0100 Subject: [PATCH 1/2] Update v3 models and add get_logs method --- .../beast/v3/_connector.py | 14 +++++++ esd_services_api_client/beast/v3/_models.py | 37 +++++++++---------- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/esd_services_api_client/beast/v3/_connector.py b/esd_services_api_client/beast/v3/_connector.py index 8f4e3fa..0f15c6a 100644 --- a/esd_services_api_client/beast/v3/_connector.py +++ b/esd_services_api_client/beast/v3/_connector.py @@ -280,3 +280,17 @@ def get_configuration( response.raise_for_status() return SparkSubmissionConfiguration.from_dict(response.json()) + + def get_logs(self, request_id: str) -> Optional[str]: + """ + Returns a deployed SparkJob configuration. + :param request_id: Submission request identifier. + :return: A job log, if found, or None + """ + response = self.http.get(f"{self.base_url}/job/logs/{request_id}") + if response.status_code == 404: + return None + if not response.ok: + response.raise_for_status() + + return "\n".join(response.json()) diff --git a/esd_services_api_client/beast/v3/_models.py b/esd_services_api_client/beast/v3/_models.py index a237b7f..9ca9b11 100644 --- a/esd_services_api_client/beast/v3/_models.py +++ b/esd_services_api_client/beast/v3/_models.py @@ -204,33 +204,18 @@ class RequestDebugMode(DataClassJsonMixin): @dataclass_json(letter_case=LetterCase.CAMEL) @dataclass -class V1TypedLocalObjectReference(DataClassJsonMixin): +class SparkSubmissionDetails(DataClassJsonMixin): """ - Reference to another SparkJob. + Job runtime details """ - api_group: str - kind: str - name: str - - -@dataclass_json(letter_case=LetterCase.CAMEL) -@dataclass -class SparkSubmissionConfiguration(DataClassJsonMixin): - """ - Configuration CRD used by Beast to run Spark apps. - """ - - root_path: str - project_name: str version: str - runnable: str execution_group: str expected_parallelism: Optional[int] flexible_driver: Optional[bool] additional_driver_node_tolerations: Dict[str, str] max_runtime_hours: Optional[int] - debug_mode: RequestDebugMode + debug_mode: Optional[RequestDebugMode] submission_mode: Optional[str] extended_code_mount: Optional[bool] submission_job_template: str @@ -239,5 +224,17 @@ class SparkSubmissionConfiguration(DataClassJsonMixin): default_arguments: Dict[str, str] inputs: List[JobSocket] outputs: List[JobSocket] - overwrite: bool - base_submission_configuration_ref: V1TypedLocalObjectReference + overwrite: Optional[bool] + + +@dataclass_json(letter_case=LetterCase.CAMEL) +@dataclass +class SparkSubmissionConfiguration(DataClassJsonMixin): + """ + Configuration CRD used by Beast to run Spark apps. + """ + + root_path: str + project_name: str + runnable: str + submission_details: SparkSubmissionDetails From 70bed7d3a5fe695f6ef9b77c5faf2c30a3b512ac Mon Sep 17 00:00:00 2001 From: George Zubrienko Date: Tue, 7 Nov 2023 13:54:21 +0100 Subject: [PATCH 2/2] Review comments --- esd_services_api_client/beast/v3/_connector.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esd_services_api_client/beast/v3/_connector.py b/esd_services_api_client/beast/v3/_connector.py index 0f15c6a..075fbef 100644 --- a/esd_services_api_client/beast/v3/_connector.py +++ b/esd_services_api_client/beast/v3/_connector.py @@ -283,7 +283,8 @@ def get_configuration( def get_logs(self, request_id: str) -> Optional[str]: """ - Returns a deployed SparkJob configuration. + Returns logs for a running or a completed submission. + :param request_id: Submission request identifier. :return: A job log, if found, or None """