From 1214959d8b1bad59e6498b4b5c111941db380643 Mon Sep 17 00:00:00 2001 From: Jaeyeon Kim Date: Thu, 19 Aug 2021 01:04:47 +0900 Subject: [PATCH 01/10] SDK: change list apis to return objects as default - change list_trials, list_experiments to return list of objects as a default - also, give 'in_short' parameter for who wants only name and status as before --- .../kubeflow/katib/api/katib_client.py | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py b/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py index 4a7212bbf5a..daad9f83d7d 100644 --- a/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py +++ b/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py @@ -245,11 +245,13 @@ def delete_experiment(self, name, namespace=None): "Exception when calling CustomObjectsApi->delete_namespaced_custom_object:\ %s\n" % e) - def list_experiments(self, namespace=None): + def list_experiments(self, namespace=None, in_short=False): """List all Katib Experiments. :param namespace: Experiments namespace. If the namespace is None, it takes "default" namespace. + :param in_short: List Experiments in summary or not + If the in_short is True, it will shows only name and status of each Experiments. :return: List of Experiment names with the statuses. :rtype: list[dict] @@ -268,12 +270,15 @@ def list_experiments(self, namespace=None): try: katibexp = thread.get(constants.APISERVER_TIMEOUT) result = [] - for i in katibexp.get("items"): - output = {} - output["name"] = i.get("metadata", {}).get("name") - output["status"] = i.get("status", {}).get( - "conditions", [])[-1].get("type") - result.append(output) + if in_short: + for i in katibexp.get("items"): + output = {} + output["name"] = i.get("metadata", {}).get("name") + output["status"] = i.get("status", {}).get( + "conditions", [])[-1].get("type") + result.append(output) + else: + result = katibexp.get("items") except multiprocessing.TimeoutError: raise RuntimeError("Timeout trying to get katib experiment.") except client.rest.ApiException as e: @@ -282,8 +287,8 @@ def list_experiments(self, namespace=None): %s\n" % e) except Exception as e: raise RuntimeError( - "There was a problem to get experiments in namespace {1}. Exception: \ - {2} ".format(namespace, e)) + "There was a problem to get experiments in namespace {0}. Exception: \ + {1} ".format(namespace, e)) return result def get_experiment_status(self, name, namespace=None): @@ -317,12 +322,14 @@ def is_experiment_succeeded(self, name, namespace=None): name, namespace=namespace) return experiment_status.lower() == "succeeded" - def list_trials(self, name=None, namespace=None): + def list_trials(self, name=None, namespace=None, in_short=False): """List all Experiment's Trials. :param name: Experiment name. :param namespace: Experiments namespace. If the namespace is None, it takes "default" namespace. + :param in_short: List Trials in summary or not + If the in_short is True, it will shows only name and status of each Trials. :return: List of Trial names with the statuses. :rtype: list[dict] @@ -341,13 +348,16 @@ def list_trials(self, name=None, namespace=None): try: katibtrial = thread.get(constants.APISERVER_TIMEOUT) result = [] - for i in katibtrial.get("items"): - output = {} - if i.get("metadata", {}).get("ownerReferences")[0].get("name") == name: - output["name"] = i.get("metadata", {}).get("name") - output["status"] = i.get("status", {}).get( - "conditions", [])[-1].get("type") - result.append(output) + if in_short: + for i in katibtrial.get("items"): + output = {} + if i.get("metadata", {}).get("ownerReferences")[0].get("name") == name: + output["name"] = i.get("metadata", {}).get("name") + output["status"] = i.get("status", {}).get( + "conditions", [])[-1].get("type") + result.append(output) + else: + result = katibtrial.get("items") except multiprocessing.TimeoutError: raise RuntimeError("Timeout trying to getkatib experiment.") except client.rest.ApiException as e: From 76b806b02160e4d8f2f08270978794489c456ac5 Mon Sep 17 00:00:00 2001 From: Jaeyeon Kim Date: Thu, 19 Aug 2021 23:16:13 +0900 Subject: [PATCH 02/10] [enh]: change return type from List[dict] to List[V1beta1Experiment] --- .../kubeflow/katib/api/katib_client.py | 63 +++++++++---------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py b/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py index daad9f83d7d..0864dcf1a93 100644 --- a/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py +++ b/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py @@ -14,10 +14,11 @@ import multiprocessing -from kubernetes import client, config - +from kubeflow.katib import V1beta1Experiment +from kubeflow.katib import V1beta1Trial from kubeflow.katib.constants import constants from kubeflow.katib.utils import utils +from kubernetes import client, config class KatibClient(object): @@ -245,16 +246,14 @@ def delete_experiment(self, name, namespace=None): "Exception when calling CustomObjectsApi->delete_namespaced_custom_object:\ %s\n" % e) - def list_experiments(self, namespace=None, in_short=False): + def list_experiments(self, namespace=None): """List all Katib Experiments. :param namespace: Experiments namespace. If the namespace is None, it takes "default" namespace. - :param in_short: List Experiments in summary or not - If the in_short is True, it will shows only name and status of each Experiments. - :return: List of Experiment names with the statuses. - :rtype: list[dict] + :return: List of Experiment objects. + :rtype: list[V1beta1Experiment] """ if namespace is None: namespace = utils.get_default_target_namespace() @@ -269,16 +268,17 @@ def list_experiments(self, namespace=None, in_short=False): katibexp = None try: katibexp = thread.get(constants.APISERVER_TIMEOUT) - result = [] - if in_short: - for i in katibexp.get("items"): - output = {} - output["name"] = i.get("metadata", {}).get("name") - output["status"] = i.get("status", {}).get( - "conditions", [])[-1].get("type") - result.append(output) - else: - result = katibexp.get("items") + result = [ + V1beta1Experiment( + api_version=item.get("apiVersion"), + kind=item.get("kind"), + metadata=item.get("metadata"), + spec=item.get("spec"), + status=item.get("status") + ) + for item in katibexp.get("items") + ] + except multiprocessing.TimeoutError: raise RuntimeError("Timeout trying to get katib experiment.") except client.rest.ApiException as e: @@ -322,17 +322,15 @@ def is_experiment_succeeded(self, name, namespace=None): name, namespace=namespace) return experiment_status.lower() == "succeeded" - def list_trials(self, name=None, namespace=None, in_short=False): + def list_trials(self, name=None, namespace=None): """List all Experiment's Trials. :param name: Experiment name. :param namespace: Experiments namespace. If the namespace is None, it takes "default" namespace. - :param in_short: List Trials in summary or not - If the in_short is True, it will shows only name and status of each Trials. - :return: List of Trial names with the statuses. - :rtype: list[dict] + :return: List of Trial objects + :rtype: list[V1beta1Trial] """ if namespace is None: namespace = utils.get_default_target_namespace() @@ -347,17 +345,16 @@ def list_trials(self, name=None, namespace=None, in_short=False): katibtrial = None try: katibtrial = thread.get(constants.APISERVER_TIMEOUT) - result = [] - if in_short: - for i in katibtrial.get("items"): - output = {} - if i.get("metadata", {}).get("ownerReferences")[0].get("name") == name: - output["name"] = i.get("metadata", {}).get("name") - output["status"] = i.get("status", {}).get( - "conditions", [])[-1].get("type") - result.append(output) - else: - result = katibtrial.get("items") + result = [ + V1beta1Trial( + api_version=item.get("apiVersion"), + kind=item.get("kind"), + metadata=item.get("metadata"), + spec=item.get("spec"), + status=item.get("status") + ) + for item in katibtrial.get("items") + ] except multiprocessing.TimeoutError: raise RuntimeError("Timeout trying to getkatib experiment.") except client.rest.ApiException as e: From 210ff0cd52e540c3fe35bf727a0015e372848268 Mon Sep 17 00:00:00 2001 From: Jaeyeon Kim Date: Sun, 22 Aug 2021 00:32:54 +0900 Subject: [PATCH 03/10] [enh]: deserialize dict to katib's custom class --- .../kubeflow/katib/api/katib_client.py | 19 ++++------------ .../v1beta1/kubeflow/katib/api_client.py | 22 ++++++++++++++++++- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py b/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py index 0864dcf1a93..149b29b6f61 100644 --- a/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py +++ b/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py @@ -14,7 +14,7 @@ import multiprocessing -from kubeflow.katib import V1beta1Experiment +from kubeflow.katib import V1beta1Experiment, api_client from kubeflow.katib import V1beta1Trial from kubeflow.katib.constants import constants from kubeflow.katib.utils import utils @@ -46,6 +46,7 @@ def __init__(self, config_file=None, context=None, self.in_cluster = True self.api_instance = client.CustomObjectsApi() + self.katib_api_client = api_client.ApiClient() def _is_ipython(self): """Returns whether we are running in notebook.""" @@ -269,13 +270,7 @@ def list_experiments(self, namespace=None): try: katibexp = thread.get(constants.APISERVER_TIMEOUT) result = [ - V1beta1Experiment( - api_version=item.get("apiVersion"), - kind=item.get("kind"), - metadata=item.get("metadata"), - spec=item.get("spec"), - status=item.get("status") - ) + self.katib_api_client.deserialize_data(item, V1beta1Experiment) for item in katibexp.get("items") ] @@ -346,13 +341,7 @@ def list_trials(self, name=None, namespace=None): try: katibtrial = thread.get(constants.APISERVER_TIMEOUT) result = [ - V1beta1Trial( - api_version=item.get("apiVersion"), - kind=item.get("kind"), - metadata=item.get("metadata"), - spec=item.get("spec"), - status=item.get("status") - ) + self.katib_api_client.deserialize_data(item, V1beta1Trial) for item in katibtrial.get("items") ] except multiprocessing.TimeoutError: diff --git a/sdk/python/v1beta1/kubeflow/katib/api_client.py b/sdk/python/v1beta1/kubeflow/katib/api_client.py index 0a77d18c53d..20397c96705 100644 --- a/sdk/python/v1beta1/kubeflow/katib/api_client.py +++ b/sdk/python/v1beta1/kubeflow/katib/api_client.py @@ -12,6 +12,8 @@ import atexit import datetime + +import kubernetes from dateutil.parser import parse import json import mimetypes @@ -22,6 +24,7 @@ # python 2 and python 3 compatibility library import six +from kubeflow.katib.rest import RESTResponse from six.moves.urllib.parse import quote from kubeflow.katib.configuration import Configuration @@ -266,6 +269,7 @@ def deserialize(self, response, response_type): :return: deserialized object. """ + assert isinstance(response, RESTResponse) # handle file downloading # save response body into a tmp file and return the instance if response_type == "file": @@ -279,6 +283,18 @@ def deserialize(self, response, response_type): return self.__deserialize(data, response_type) + def deserialize_data(self, data, data_type): + """Deserializes data into an object. + + :param data: object to be deserialized. + :param data_type: class literal for + deserialized object, or string of class name. + + :return: deserialized object. + """ + assert isinstance(data, (dict, list, str)) + return self.__deserialize(data, data_type) + def __deserialize(self, data, klass): """Deserializes dict, list, str into an object. @@ -304,8 +320,12 @@ def __deserialize(self, data, klass): # convert str to class if klass in self.NATIVE_TYPES_MAPPING: klass = self.NATIVE_TYPES_MAPPING[klass] - else: + elif klass in dir(kubeflow.katib.models): klass = getattr(kubeflow.katib.models, klass) + elif klass in dir(kubernetes.client.models): + klass = getattr(kubernetes.client.models, klass) + else: + raise ValueError(f"type: {klass} is not supported to deserialized") if klass in self.PRIMITIVE_TYPES: return self.__deserialize_primitive(data, klass) From c16ba72641c917faa6e8c57a4e41fe47a42c9cfb Mon Sep 17 00:00:00 2001 From: Jaeyeon Kim Date: Sun, 22 Aug 2021 00:40:30 +0900 Subject: [PATCH 04/10] [docs]: refactor KatibClient docs --- sdk/python/v1beta1/docs/KatibClient.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/python/v1beta1/docs/KatibClient.md b/sdk/python/v1beta1/docs/KatibClient.md index b438b47c4c1..b09c901aeaa 100644 --- a/sdk/python/v1beta1/docs/KatibClient.md +++ b/sdk/python/v1beta1/docs/KatibClient.md @@ -116,7 +116,7 @@ dict List all Katib Experiments. If the namespace is `None`, it takes "default" namespace. -Return list of Experiment names with the statuses. +Return list of Experiment objects. ### Parameters @@ -126,7 +126,7 @@ Return list of Experiment names with the statuses. ### Return type -list[dict] +list[V1beta1Experiment] ## get_experiment_status @@ -175,7 +175,7 @@ bool List all Experiment's Trials. If the namespace is `None`, it takes "default" namespace. -Return list of Trial names with the statuses. +Return list of Trial objects ### Parameters @@ -186,7 +186,7 @@ Return list of Trial names with the statuses. ### Return type -list[dict] +list[V1beta1Trial] ## get_success_trial_details From ecf9e64a8923be89df1eff388f4cc685d4dd588a Mon Sep 17 00:00:00 2001 From: Jaeyeon Kim Date: Sat, 28 Aug 2021 21:11:07 +0900 Subject: [PATCH 05/10] change deserialize method location to utils --- .../kubeflow/katib/api/katib_client.py | 6 +- .../v1beta1/kubeflow/katib/api_client.py | 22 +-- .../v1beta1/kubeflow/katib/utils/utils.py | 174 ++++++++++++++++++ 3 files changed, 178 insertions(+), 24 deletions(-) diff --git a/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py b/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py index 149b29b6f61..1fb2953e11e 100644 --- a/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py +++ b/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py @@ -46,7 +46,7 @@ def __init__(self, config_file=None, context=None, self.in_cluster = True self.api_instance = client.CustomObjectsApi() - self.katib_api_client = api_client.ApiClient() + self.deserializer = utils.Deserializer() def _is_ipython(self): """Returns whether we are running in notebook.""" @@ -270,7 +270,7 @@ def list_experiments(self, namespace=None): try: katibexp = thread.get(constants.APISERVER_TIMEOUT) result = [ - self.katib_api_client.deserialize_data(item, V1beta1Experiment) + self.deserializer.deserialize(item, V1beta1Experiment) for item in katibexp.get("items") ] @@ -341,7 +341,7 @@ def list_trials(self, name=None, namespace=None): try: katibtrial = thread.get(constants.APISERVER_TIMEOUT) result = [ - self.katib_api_client.deserialize_data(item, V1beta1Trial) + self.deserializer.deserialize(item, V1beta1Trial) for item in katibtrial.get("items") ] except multiprocessing.TimeoutError: diff --git a/sdk/python/v1beta1/kubeflow/katib/api_client.py b/sdk/python/v1beta1/kubeflow/katib/api_client.py index 20397c96705..0a77d18c53d 100644 --- a/sdk/python/v1beta1/kubeflow/katib/api_client.py +++ b/sdk/python/v1beta1/kubeflow/katib/api_client.py @@ -12,8 +12,6 @@ import atexit import datetime - -import kubernetes from dateutil.parser import parse import json import mimetypes @@ -24,7 +22,6 @@ # python 2 and python 3 compatibility library import six -from kubeflow.katib.rest import RESTResponse from six.moves.urllib.parse import quote from kubeflow.katib.configuration import Configuration @@ -269,7 +266,6 @@ def deserialize(self, response, response_type): :return: deserialized object. """ - assert isinstance(response, RESTResponse) # handle file downloading # save response body into a tmp file and return the instance if response_type == "file": @@ -283,18 +279,6 @@ def deserialize(self, response, response_type): return self.__deserialize(data, response_type) - def deserialize_data(self, data, data_type): - """Deserializes data into an object. - - :param data: object to be deserialized. - :param data_type: class literal for - deserialized object, or string of class name. - - :return: deserialized object. - """ - assert isinstance(data, (dict, list, str)) - return self.__deserialize(data, data_type) - def __deserialize(self, data, klass): """Deserializes dict, list, str into an object. @@ -320,12 +304,8 @@ def __deserialize(self, data, klass): # convert str to class if klass in self.NATIVE_TYPES_MAPPING: klass = self.NATIVE_TYPES_MAPPING[klass] - elif klass in dir(kubeflow.katib.models): - klass = getattr(kubeflow.katib.models, klass) - elif klass in dir(kubernetes.client.models): - klass = getattr(kubernetes.client.models, klass) else: - raise ValueError(f"type: {klass} is not supported to deserialized") + klass = getattr(kubeflow.katib.models, klass) if klass in self.PRIMITIVE_TYPES: return self.__deserialize_primitive(data, klass) diff --git a/sdk/python/v1beta1/kubeflow/katib/utils/utils.py b/sdk/python/v1beta1/kubeflow/katib/utils/utils.py index 537cfa2c7fc..d192ddb00af 100644 --- a/sdk/python/v1beta1/kubeflow/katib/utils/utils.py +++ b/sdk/python/v1beta1/kubeflow/katib/utils/utils.py @@ -12,7 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import absolute_import + +import datetime import os +import re + +import kubeflow.katib.models +import kubernetes.client.models +# python 2 and python 3 compatibility library +import six +from dateutil.parser import parse +from kubeflow.katib import rest def is_running_in_k8s(): @@ -34,3 +45,166 @@ def set_katib_namespace(katib): katib_namespace = katib.metadata.namespace namespace = katib_namespace or get_default_target_namespace() return namespace + + +class Deserializer: + """Deserializer for deserializing data into katib's custom objects. + """ + PRIMITIVE_TYPES = (float, bool, bytes, six.text_type) + six.integer_types + NATIVE_TYPES_MAPPING = { + 'int': int, + 'long': int if six.PY3 else long, # noqa: F821 + 'float': float, + 'str': str, + 'bool': bool, + 'date': datetime.date, + 'datetime': datetime.datetime, + 'object': object, + } + + def deserialize(self, data, data_type): + """Deserializes data into an object. + + :param data: object to be deserialized. + :param data_type: class literal for + deserialized object, or string of class name. + + :return: deserialized object. + """ + assert isinstance(data, (dict, list, str)) + + return self.__deserialize(data, data_type) + + def __deserialize(self, data, klass): + """Deserializes dict, list, str into an object. + + :param data: dict, list or str. + :param klass: class literal, or string of class name. + + :return: object. + """ + if data is None: + return None + + if type(klass) == str: + if klass.startswith('list['): + sub_kls = re.match(r'list\[(.*)\]', klass).group(1) + return [self.__deserialize(sub_data, sub_kls) + for sub_data in data] + + if klass.startswith('dict('): + sub_kls = re.match(r'dict\(([^,]*), (.*)\)', klass).group(2) + return {k: self.__deserialize(v, sub_kls) + for k, v in six.iteritems(data)} + + # convert str to class + if klass in self.NATIVE_TYPES_MAPPING: + klass = self.NATIVE_TYPES_MAPPING[klass] + elif klass in dir(kubeflow.katib.models): + klass = getattr(kubeflow.katib.models, klass) + elif klass in dir(kubernetes.client.models): + klass = getattr(kubernetes.client.models, klass) + else: + raise ValueError(f"type: {klass} is not supported to deserialized") + + if klass in self.PRIMITIVE_TYPES: + return self.__deserialize_primitive(data, klass) + elif klass == object: + return self.__deserialize_object(data) + elif klass == datetime.date: + return self.__deserialize_date(data) + elif klass == datetime.datetime: + return self.__deserialize_datetime(data) + else: + return self.__deserialize_model(data, klass) + + def __deserialize_primitive(self, data, klass): + """Deserializes string to primitive type. + + :param data: str. + :param klass: class literal. + + :return: int, long, float, str, bool. + """ + try: + return klass(data) + except UnicodeEncodeError: + return six.text_type(data) + except TypeError: + return data + + def __deserialize_object(self, value): + """Return an original value. + + :return: object. + """ + return value + + def __deserialize_date(self, string): + """Deserializes string to date. + + :param string: str. + :return: date. + """ + try: + return parse(string).date() + except ImportError: + return string + except ValueError: + raise rest.ApiException( + status=0, + reason="Failed to parse `{0}` as date object".format(string) + ) + + def __deserialize_datetime(self, string): + """Deserializes string to datetime. + + The string should be in iso8601 datetime format. + + :param string: str. + :return: datetime. + """ + try: + return parse(string) + except ImportError: + return string + except ValueError: + raise rest.ApiException( + status=0, + reason=( + "Failed to parse `{0}` as datetime object" + .format(string) + ) + ) + + def __deserialize_model(self, data, klass): + """Deserializes list or dict to model. + + :param data: dict, list. + :param klass: class literal. + :return: model object. + """ + has_discriminator = False + if (hasattr(klass, 'get_real_child_model') + and klass.discriminator_value_class_map): + has_discriminator = True + + if not klass.openapi_types and has_discriminator is False: + return data + + kwargs = {} + if (data is not None and + klass.openapi_types is not None and + isinstance(data, (list, dict))): + for attr, attr_type in six.iteritems(klass.openapi_types): + if klass.attribute_map[attr] in data: + value = data[klass.attribute_map[attr]] + kwargs[attr] = self.__deserialize(value, attr_type) + + instance = klass(**kwargs) + + if has_discriminator: + klass_name = instance.get_real_child_model(data) + if klass_name: + instance = self.__deserialize(data, klass_name) + return instance From 979b2d2d496a4e9ba7b417a3ca198bc64f7fe559 Mon Sep 17 00:00:00 2001 From: Jaeyeon Kim Date: Sat, 28 Aug 2021 21:12:40 +0900 Subject: [PATCH 06/10] remove useless import --- sdk/python/v1beta1/kubeflow/katib/api/katib_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py b/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py index 1fb2953e11e..d42ba63bea1 100644 --- a/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py +++ b/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py @@ -14,7 +14,7 @@ import multiprocessing -from kubeflow.katib import V1beta1Experiment, api_client +from kubeflow.katib import V1beta1Experiment from kubeflow.katib import V1beta1Trial from kubeflow.katib.constants import constants from kubeflow.katib.utils import utils From 4a453d8e1e2a4ff390b4691811f0a71c6c2d7366 Mon Sep 17 00:00:00 2001 From: Jaeyeon Kim Date: Sat, 28 Aug 2021 22:56:41 +0900 Subject: [PATCH 07/10] Add objects necessary to deserilization in swagger --- hack/gen-python-sdk/post_gen.py | 2 ++ hack/gen-python-sdk/swagger_config.json | 4 +++- sdk/python/v1beta1/kubeflow/katib/models/__init__.py | 2 ++ sdk/python/v1beta1/kubeflow/katib/utils/utils.py | 3 --- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/hack/gen-python-sdk/post_gen.py b/hack/gen-python-sdk/post_gen.py index e57857410d2..d5b6c71c6d1 100644 --- a/hack/gen-python-sdk/post_gen.py +++ b/hack/gen-python-sdk/post_gen.py @@ -50,6 +50,8 @@ def _rewrite_helper(input_file, output_file, rewrite_rules): lines.append("from kubernetes.client import V1ListMeta\n") lines.append("from kubernetes.client import V1Container\n") lines.append("from kubernetes.client import V1HTTPGetAction\n") + lines.append("from kubernetes.client import V1ManagedFieldsEntry\n") + lines.append("from kubernetes.client import V1OwnerReference\n") with open(output_file, 'w') as f: f.writelines(lines) diff --git a/hack/gen-python-sdk/swagger_config.json b/hack/gen-python-sdk/swagger_config.json index ebdfcf1faec..4e02deb3db5 100644 --- a/hack/gen-python-sdk/swagger_config.json +++ b/hack/gen-python-sdk/swagger_config.json @@ -6,7 +6,9 @@ "V1Container": "from kubernetes.client import V1Container", "V1ListMeta": "from kubernetes.client import V1ListMeta", "V1ObjectMeta": "from kubernetes.client import V1ObjectMeta", - "V1HTTPGetAction": "from kubernetes.client import V1HTTPGetAction" + "V1HTTPGetAction": "from kubernetes.client import V1HTTPGetAction", + "V1ManagedFieldsEntry": "from kubernetes.client import V1ManagedFieldsEntry", + "V1OwnerReference": "from kubernetes.client import V1OwnerReference" }, "typeMappings": { "V1Time": "datetime", diff --git a/sdk/python/v1beta1/kubeflow/katib/models/__init__.py b/sdk/python/v1beta1/kubeflow/katib/models/__init__.py index 00320adcba2..c43cbd34505 100644 --- a/sdk/python/v1beta1/kubeflow/katib/models/__init__.py +++ b/sdk/python/v1beta1/kubeflow/katib/models/__init__.py @@ -61,3 +61,5 @@ from kubernetes.client import V1ListMeta from kubernetes.client import V1Container from kubernetes.client import V1HTTPGetAction +from kubernetes.client import V1ManagedFieldsEntry +from kubernetes.client import V1OwnerReference diff --git a/sdk/python/v1beta1/kubeflow/katib/utils/utils.py b/sdk/python/v1beta1/kubeflow/katib/utils/utils.py index d192ddb00af..0a9f8bb993d 100644 --- a/sdk/python/v1beta1/kubeflow/katib/utils/utils.py +++ b/sdk/python/v1beta1/kubeflow/katib/utils/utils.py @@ -19,7 +19,6 @@ import re import kubeflow.katib.models -import kubernetes.client.models # python 2 and python 3 compatibility library import six from dateutil.parser import parse @@ -102,8 +101,6 @@ def __deserialize(self, data, klass): klass = self.NATIVE_TYPES_MAPPING[klass] elif klass in dir(kubeflow.katib.models): klass = getattr(kubeflow.katib.models, klass) - elif klass in dir(kubernetes.client.models): - klass = getattr(kubernetes.client.models, klass) else: raise ValueError(f"type: {klass} is not supported to deserialized") From 3c98e2b8e878e9483099cbaf4936a18f0db2aa87 Mon Sep 17 00:00:00 2001 From: Jaeyeon Kim Date: Sun, 29 Aug 2021 00:36:03 +0900 Subject: [PATCH 08/10] use fakeresponse rather than duplicating codes --- .../kubeflow/katib/api/katib_client.py | 7 +- .../v1beta1/kubeflow/katib/utils/utils.py | 173 +----------------- 2 files changed, 12 insertions(+), 168 deletions(-) diff --git a/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py b/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py index d42ba63bea1..f74646ee022 100644 --- a/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py +++ b/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py @@ -16,6 +16,7 @@ from kubeflow.katib import V1beta1Experiment from kubeflow.katib import V1beta1Trial +from kubeflow.katib.api_client import ApiClient from kubeflow.katib.constants import constants from kubeflow.katib.utils import utils from kubernetes import client, config @@ -46,7 +47,7 @@ def __init__(self, config_file=None, context=None, self.in_cluster = True self.api_instance = client.CustomObjectsApi() - self.deserializer = utils.Deserializer() + self.api_client = ApiClient() def _is_ipython(self): """Returns whether we are running in notebook.""" @@ -270,7 +271,7 @@ def list_experiments(self, namespace=None): try: katibexp = thread.get(constants.APISERVER_TIMEOUT) result = [ - self.deserializer.deserialize(item, V1beta1Experiment) + self.api_client.deserialize(utils.FakeResponse(item), V1beta1Experiment) for item in katibexp.get("items") ] @@ -341,7 +342,7 @@ def list_trials(self, name=None, namespace=None): try: katibtrial = thread.get(constants.APISERVER_TIMEOUT) result = [ - self.deserializer.deserialize(item, V1beta1Trial) + self.api_client.deserialize(utils.FakeResponse(item), V1beta1Trial) for item in katibtrial.get("items") ] except multiprocessing.TimeoutError: diff --git a/sdk/python/v1beta1/kubeflow/katib/utils/utils.py b/sdk/python/v1beta1/kubeflow/katib/utils/utils.py index 0a9f8bb993d..428eb9fa83d 100644 --- a/sdk/python/v1beta1/kubeflow/katib/utils/utils.py +++ b/sdk/python/v1beta1/kubeflow/katib/utils/utils.py @@ -14,15 +14,11 @@ from __future__ import absolute_import -import datetime +import json import os -import re -import kubeflow.katib.models + # python 2 and python 3 compatibility library -import six -from dateutil.parser import parse -from kubeflow.katib import rest def is_running_in_k8s(): @@ -45,163 +41,10 @@ def set_katib_namespace(katib): namespace = katib_namespace or get_default_target_namespace() return namespace - -class Deserializer: - """Deserializer for deserializing data into katib's custom objects. +class FakeResponse: + """Fake object of RESTResponse to deserialize + Ref) https://github.com/kubeflow/katib/pull/1630#discussion_r697877815 + Ref) https://github.com/kubernetes-client/python/issues/977#issuecomment-592030030 """ - PRIMITIVE_TYPES = (float, bool, bytes, six.text_type) + six.integer_types - NATIVE_TYPES_MAPPING = { - 'int': int, - 'long': int if six.PY3 else long, # noqa: F821 - 'float': float, - 'str': str, - 'bool': bool, - 'date': datetime.date, - 'datetime': datetime.datetime, - 'object': object, - } - - def deserialize(self, data, data_type): - """Deserializes data into an object. - - :param data: object to be deserialized. - :param data_type: class literal for - deserialized object, or string of class name. - - :return: deserialized object. - """ - assert isinstance(data, (dict, list, str)) - - return self.__deserialize(data, data_type) - - def __deserialize(self, data, klass): - """Deserializes dict, list, str into an object. - - :param data: dict, list or str. - :param klass: class literal, or string of class name. - - :return: object. - """ - if data is None: - return None - - if type(klass) == str: - if klass.startswith('list['): - sub_kls = re.match(r'list\[(.*)\]', klass).group(1) - return [self.__deserialize(sub_data, sub_kls) - for sub_data in data] - - if klass.startswith('dict('): - sub_kls = re.match(r'dict\(([^,]*), (.*)\)', klass).group(2) - return {k: self.__deserialize(v, sub_kls) - for k, v in six.iteritems(data)} - - # convert str to class - if klass in self.NATIVE_TYPES_MAPPING: - klass = self.NATIVE_TYPES_MAPPING[klass] - elif klass in dir(kubeflow.katib.models): - klass = getattr(kubeflow.katib.models, klass) - else: - raise ValueError(f"type: {klass} is not supported to deserialized") - - if klass in self.PRIMITIVE_TYPES: - return self.__deserialize_primitive(data, klass) - elif klass == object: - return self.__deserialize_object(data) - elif klass == datetime.date: - return self.__deserialize_date(data) - elif klass == datetime.datetime: - return self.__deserialize_datetime(data) - else: - return self.__deserialize_model(data, klass) - - def __deserialize_primitive(self, data, klass): - """Deserializes string to primitive type. - - :param data: str. - :param klass: class literal. - - :return: int, long, float, str, bool. - """ - try: - return klass(data) - except UnicodeEncodeError: - return six.text_type(data) - except TypeError: - return data - - def __deserialize_object(self, value): - """Return an original value. - - :return: object. - """ - return value - - def __deserialize_date(self, string): - """Deserializes string to date. - - :param string: str. - :return: date. - """ - try: - return parse(string).date() - except ImportError: - return string - except ValueError: - raise rest.ApiException( - status=0, - reason="Failed to parse `{0}` as date object".format(string) - ) - - def __deserialize_datetime(self, string): - """Deserializes string to datetime. - - The string should be in iso8601 datetime format. - - :param string: str. - :return: datetime. - """ - try: - return parse(string) - except ImportError: - return string - except ValueError: - raise rest.ApiException( - status=0, - reason=( - "Failed to parse `{0}` as datetime object" - .format(string) - ) - ) - - def __deserialize_model(self, data, klass): - """Deserializes list or dict to model. - - :param data: dict, list. - :param klass: class literal. - :return: model object. - """ - has_discriminator = False - if (hasattr(klass, 'get_real_child_model') - and klass.discriminator_value_class_map): - has_discriminator = True - - if not klass.openapi_types and has_discriminator is False: - return data - - kwargs = {} - if (data is not None and - klass.openapi_types is not None and - isinstance(data, (list, dict))): - for attr, attr_type in six.iteritems(klass.openapi_types): - if klass.attribute_map[attr] in data: - value = data[klass.attribute_map[attr]] - kwargs[attr] = self.__deserialize(value, attr_type) - - instance = klass(**kwargs) - - if has_discriminator: - klass_name = instance.get_real_child_model(data) - if klass_name: - instance = self.__deserialize(data, klass_name) - return instance + def __init__(self, obj): + self.data = json.dumps(obj) From 175150623a4721a058de8dac8cd3eb2898504271 Mon Sep 17 00:00:00 2001 From: Jaeyeon Kim Date: Sun, 29 Aug 2021 01:30:38 +0900 Subject: [PATCH 09/10] Update sdk/python/v1beta1/kubeflow/katib/utils/utils.py Co-authored-by: Andrey Velichkevich --- sdk/python/v1beta1/kubeflow/katib/utils/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sdk/python/v1beta1/kubeflow/katib/utils/utils.py b/sdk/python/v1beta1/kubeflow/katib/utils/utils.py index 428eb9fa83d..8867cb344dd 100644 --- a/sdk/python/v1beta1/kubeflow/katib/utils/utils.py +++ b/sdk/python/v1beta1/kubeflow/katib/utils/utils.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import - import json import os From 88570a1cecda092c3cbba5e008ad66cbb63b9db9 Mon Sep 17 00:00:00 2001 From: Jaeyeon Kim Date: Sun, 29 Aug 2021 01:31:06 +0900 Subject: [PATCH 10/10] Update sdk/python/v1beta1/kubeflow/katib/utils/utils.py Co-authored-by: Andrey Velichkevich --- sdk/python/v1beta1/kubeflow/katib/utils/utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/sdk/python/v1beta1/kubeflow/katib/utils/utils.py b/sdk/python/v1beta1/kubeflow/katib/utils/utils.py index 8867cb344dd..08370eeaa7c 100644 --- a/sdk/python/v1beta1/kubeflow/katib/utils/utils.py +++ b/sdk/python/v1beta1/kubeflow/katib/utils/utils.py @@ -16,9 +16,6 @@ import os -# python 2 and python 3 compatibility library - - def is_running_in_k8s(): return os.path.isdir('/var/run/secrets/kubernetes.io/')