From 583bc73cf6ab5cbbd549fa30c2c62ba048a74bb7 Mon Sep 17 00:00:00 2001 From: Kevin REMY Date: Mon, 25 Nov 2024 12:09:56 +0100 Subject: [PATCH] Publish 13.2.3 on PyPI (#311) * Sync sources * Release metadata * Update release date --- HISTORY.txt | 6 ++++++ dataikuapi/dss/admin.py | 10 +++++----- dataikuapi/dss/analysis.py | 1 - dataikuapi/dss/future.py | 8 +++++--- dataikuapi/dss/job.py | 8 +++----- dataikuapi/dss/ml.py | 8 +++++--- dataikuapi/dss/scenario.py | 13 ++++++++----- dataikuapi/dssclient.py | 4 +--- dataikuapi/utils.py | 33 ++++++++++++++++++++++++++++++++- setup.py | 2 +- 10 files changed, 66 insertions(+), 27 deletions(-) diff --git a/HISTORY.txt b/HISTORY.txt index 569f64cd..a44e5970 100644 --- a/HISTORY.txt +++ b/HISTORY.txt @@ -1,6 +1,12 @@ Changelog ========== + +13.2.3 (2024-11-25) +--------------------- + +* Initial release for DSS 13.2.3 + 13.2.2.1 (2024-11-06) --------------------- diff --git a/dataikuapi/dss/admin.py b/dataikuapi/dss/admin.py index 1dae0037..2ebfb5e3 100644 --- a/dataikuapi/dss/admin.py +++ b/dataikuapi/dss/admin.py @@ -1303,7 +1303,7 @@ def delete(self): .. note:: - This call requires an API key with admin rights + This call requires an API key with `Manage all code envs` permission """ resp = self.client._perform_json( "DELETE", "/admin/code-envs/%s/%s" % (self.env_lang, self.env_name)) @@ -1328,7 +1328,7 @@ def get_definition(self): .. note:: - This call requires an API key with admin rights + This call requires an API key with `Create code envs` or `Manage all code envs` permission :return: the code env definition :rtype: dict @@ -1364,7 +1364,7 @@ def set_definition(self, env): .. note:: - This call requires an API key with admin rights + This call requires an API key with `Create code envs` or `Manage all code envs` permission .. important:: @@ -1441,7 +1441,7 @@ def set_jupyter_support(self, active): .. note:: - This call requires an API key with admin rights + This call requires an API key with `Create code envs` or `Manage all code envs` permission :param boolean active: True to activate jupyter support, False to deactivate """ @@ -1460,7 +1460,7 @@ def update_packages(self, force_rebuild_env=False): .. note:: - This call requires an API key with admin rights + This call requires an API key with `Create code envs` or `Manage all code envs` permission :param boolean force_rebuild_env: whether to rebuild the code env from scratch diff --git a/dataikuapi/dss/analysis.py b/dataikuapi/dss/analysis.py index e9acae35..d06e8beb 100644 --- a/dataikuapi/dss/analysis.py +++ b/dataikuapi/dss/analysis.py @@ -2,7 +2,6 @@ from ..utils import DataikuUTF8CSVReader from ..utils import DataikuStreamedHttpUTF8CSVReader import json -import time from .metrics import ComputedMetrics from .ml import DSSMLTask from .utils import DSSDatasetSelectionBuilder diff --git a/dataikuapi/dss/future.py b/dataikuapi/dss/future.py index a5f7c330..53874e3d 100644 --- a/dataikuapi/dss/future.py +++ b/dataikuapi/dss/future.py @@ -1,5 +1,4 @@ -import time - +from ..utils import _ExponentialBackoff class DSSFuture(object): """ @@ -126,8 +125,11 @@ def wait_for_result(self): return self.result_wrapper(self.state.get('result', None)) if self.state is None or not self.state.get('hasResult', False) or self.state_is_peek: self.get_state() + + eb = _ExponentialBackoff() + while not self.state.get('hasResult', False): - time.sleep(5) + eb.sleep_next() self.get_state() if self.state.get('hasResult', False): return self.result_wrapper(self.state.get('result', None)) diff --git a/dataikuapi/dss/job.py b/dataikuapi/dss/job.py index 4e263cd9..0199a5af 100644 --- a/dataikuapi/dss/job.py +++ b/dataikuapi/dss/job.py @@ -1,7 +1,6 @@ -import time import sys from ..utils import DataikuException - +from ..utils import _ExponentialBackoff class DSSJob(object): """ @@ -77,10 +76,9 @@ def wait(self, no_fail=False): :rtype: dict """ job_state = self.job.get_status().get("baseStatus", {}).get("state", "") - sleep_time = 2 + eb = _ExponentialBackoff() while job_state not in ["DONE", "ABORTED", "FAILED"]: - sleep_time = 60 if sleep_time >= 60 else sleep_time * 1.2 - time.sleep(int(sleep_time)) + eb.sleep_next() job_state = self.job.get_status().get("baseStatus", {}).get("state", "") if no_fail or (job_state == "DONE"): diff --git a/dataikuapi/dss/ml.py b/dataikuapi/dss/ml.py index aa4eb63d..ffebbfff 100644 --- a/dataikuapi/dss/ml.py +++ b/dataikuapi/dss/ml.py @@ -1,7 +1,6 @@ import json import logging import re -import time import warnings from six import string_types @@ -11,6 +10,7 @@ from .utils import DSSFilterBuilder from ..utils import DataikuException from ..utils import _write_response_content_to_file +from ..utils import _ExponentialBackoff logger = logging.getLogger("dataikuapi.dss.ml") @@ -4486,11 +4486,12 @@ def wait_guess_complete(self): This should be called immediately after the creation of a new ML Task if the ML Task was created with ``wait_guess_complete = False``, before calling :meth:`get_settings` or :meth:`train`. """ + eb = _ExponentialBackoff() while True: status = self.get_status() if status.get("guessing", "???") == False: break - time.sleep(0.2) + eb.sleep_next() def get_status(self): """ @@ -4627,11 +4628,12 @@ def wait_train_complete(self): To be used following any asynchronous training started with :meth:`start_train` or :meth:`start_ensembling` """ + eb = _ExponentialBackoff() while True: status = self.get_status() if status.get("training", "???") == False: break - time.sleep(2) + eb.sleep_next() def get_trained_models_ids(self, session_id=None, algorithm=None): """ diff --git a/dataikuapi/dss/scenario.py b/dataikuapi/dss/scenario.py index 6e0f1191..69cab537 100644 --- a/dataikuapi/dss/scenario.py +++ b/dataikuapi/dss/scenario.py @@ -1,6 +1,6 @@ from datetime import datetime -import time, warnings -from ..utils import DataikuException, _timestamp_ms_to_zoned_datetime, _local_timezone +import warnings +from ..utils import DataikuException, _timestamp_ms_to_zoned_datetime, _local_timezone, _ExponentialBackoff from .discussion import DSSObjectDiscussions from .utils import DSSTaggableObjectListItem from dateutil.tz import tzlocal @@ -715,9 +715,10 @@ def wait_for_completion(self, no_fail=False): :param boolean no_fail: if False, raises an exception if scenario fails """ + eb = _ExponentialBackoff() while self.running: self.refresh() - time.sleep(5) + eb.sleep_next() if self.outcome != 'SUCCESS' and no_fail == False: raise DataikuException("Scenario run returned status %s" % outcome) @@ -963,9 +964,10 @@ def wait(self, no_fail=False): :return: the final state of the scenario run (see :meth:`DSSScenarioRun.get_info()`) :rtype: dict """ + eb = _ExponentialBackoff() while not self.scenario_run.run.get('result', False): self.scenario_run = self.trigger_fire.get_scenario_run() - time.sleep(5) + eb.sleep_next() outcome = self.scenario_run.run.get('result', None).get('outcome', 'UNKNOWN') if outcome == 'SUCCESS' or no_fail: return self.scenario_run @@ -1012,6 +1014,7 @@ def wait_for_scenario_run(self, no_fail=False): """ scenario_run = None refresh_trigger_counter = 0 + eb = _ExponentialBackoff() while scenario_run is None: refresh_trigger_counter += 1 if refresh_trigger_counter == 10: @@ -1022,7 +1025,7 @@ def wait_for_scenario_run(self, no_fail=False): else: raise DataikuException("Scenario run has been cancelled") scenario_run = self.get_scenario_run() - time.sleep(5) + eb.sleep_next() return scenario_run def get_scenario_run(self): diff --git a/dataikuapi/dssclient.py b/dataikuapi/dssclient.py index edc0a49d..2ecdc382 100644 --- a/dataikuapi/dssclient.py +++ b/dataikuapi/dssclient.py @@ -663,8 +663,6 @@ def list_code_envs(self, as_objects=False): """ List all code envs setup on the DSS instance - Note: this call requires an API key with admin rights - :param boolean as_objects: if True, each returned item will be a :class:`dataikuapi.dss.future.DSSCodeEnv` :returns: a list of code envs. Each code env is a dict containing at least "name", "type" and "language" """ @@ -689,7 +687,7 @@ def create_code_env(self, env_lang, env_name, deployment_mode, params=None): """ Create a code env, and return a handle to interact with it - Note: this call requires an API key with admin rights + Note: this call requires an API key with `Create code envs` or `Manage all code envs` permission :param env_lang: the language (PYTHON or R) of the new code env :param env_name: the name of the new code env diff --git a/dataikuapi/utils.py b/dataikuapi/utils.py index d2ddf13e..b0a622b0 100644 --- a/dataikuapi/utils.py +++ b/dataikuapi/utils.py @@ -5,6 +5,7 @@ import zipfile import itertools import sys +import time from datetime import datetime if sys.version_info > (3,0): @@ -159,4 +160,34 @@ def _timestamp_ms_to_zoned_datetime(timestamp_ms): if timestamp_ms and timestamp_ms > 0: return datetime.fromtimestamp(timestamp_ms / 1000, tz=_local_timezone) else: - return None \ No newline at end of file + return None + + +class _ExponentialBackoff(object): + # These values lead to: + # - 25 calls in the first 20 seconds + # - Overhead overall remains below 10% + # - After 1 minute, the interval is 5 seconds + + def __init__(self, initial_time_ms=200, max_time_ms=15000, factor=1.1): + self.initial_time_ms = initial_time_ms + self.last_time = None + self.max_time_ms = max_time_ms + self.factor = factor + + def next_sleep_time(self): + if self.last_time is None: + next_time = self.initial_time_ms + else: + next_time = self.last_time * self.factor + if next_time > self.max_time_ms: + next_time = self.max_time_ms + + self.last_time = next_time + + return next_time + + def sleep_next(self): + sleep_time = float(self.next_sleep_time()) / 1000.0 + #print("Sleeping %.3f" % sleep_time) + time.sleep(sleep_time) \ No newline at end of file diff --git a/setup.py b/setup.py index 9006c89f..59fda1c6 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup -VERSION = "13.2.2.1" +VERSION = "13.2.3" long_description = (open('README').read() + '\n\n' + open('HISTORY.txt').read())