diff --git a/ivy/utils/binaries.py b/ivy/utils/binaries.py index faf30692ba1c8..2c922637ada4e 100644 --- a/ivy/utils/binaries.py +++ b/ivy/utils/binaries.py @@ -3,6 +3,7 @@ import json from pip._vendor.packaging import tags from urllib import request +from tqdm import tqdm def _get_paths_from_binaries(binaries, root_dir=""): @@ -68,38 +69,44 @@ def cleanup_and_fetch_binaries(clean=True): # clean up existing binaries if clean: - print("Cleaning up existing binaries...") + print("Cleaning up existing binaries", end="\r") for root, _, files in os.walk(folder_path, topdown=True): for file in files: if file.split(".")[-1] in binaries_exts: os.remove(os.path.join(root, file)) + print("Cleaning up existing binaries --> done") - print("Downloading new binaries...") + print("Downloading new binaries") all_tags = list(tags.sys_tags()) version = os.environ["VERSION"] if "VERSION" in os.environ else "main" terminate = False # download binaries for the tag with highest precedence - for tag in all_tags: - if terminate: - break - for path in binaries_paths: - module = path[len(folder_path) :][1:].split(os.sep)[1] - if os.path.exists(path) or str(tag) not in available_configs[module]: - continue - folders = path.split(os.sep) - _, file_path = os.sep.join(folders[:-1]), folders[-1] - file_name = f"{file_path[:-3]}_{tag}.so" - search_path = f"{module}/{file_name}" - try: - response = request.urlopen( - "https://github.com/unifyai/binaries/raw/" - f"{version}/{search_path}", - timeout=40, - ) - os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, "wb") as f: - f.write(response.read()) - terminate = path == binaries_paths[-1] - except request.HTTPError: + with tqdm(total=len(binaries_paths)) as pbar: + for tag in all_tags: + if terminate: break + for path in binaries_paths: + module = path[len(folder_path) :][1:].split(os.sep)[1] + if ( + os.path.exists(path) + or str(tag) not in available_configs[module] + ): + continue + folders = path.split(os.sep) + _, file_path = os.sep.join(folders[:-1]), folders[-1] + file_name = f"{file_path[:-3]}_{tag}.so" + search_path = f"{module}/{file_name}" + try: + response = request.urlopen( + "https://github.com/unifyai/binaries/raw/" + f"{version}/{search_path}", + timeout=40, + ) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as f: + f.write(response.read()) + terminate = path == binaries_paths[-1] + pbar.update(1) + except request.HTTPError: + break diff --git a/requirements/optional.txt b/requirements/optional.txt index 0cc2e46556f9b..e34585650624e 100644 --- a/requirements/optional.txt +++ b/requirements/optional.txt @@ -17,7 +17,6 @@ scipy # unpinned dm-haiku # unpinned mod_name=haiku flax pydriller -tqdm coverage scikit-learn # mod_name=sklearn pandas diff --git a/requirements/optional_apple_silicon_2.txt b/requirements/optional_apple_silicon_2.txt index 42a07692d8343..7a3325ede80c3 100644 --- a/requirements/optional_apple_silicon_2.txt +++ b/requirements/optional_apple_silicon_2.txt @@ -5,7 +5,6 @@ dm-haiku # mod_name=haiku flax protobuf pydriller -tqdm coverage scikit-learn # mod_name=sklearn pandas diff --git a/requirements/optional_apple_silicon_gpu_2.txt b/requirements/optional_apple_silicon_gpu_2.txt index 42a07692d8343..7a3325ede80c3 100644 --- a/requirements/optional_apple_silicon_gpu_2.txt +++ b/requirements/optional_apple_silicon_gpu_2.txt @@ -5,7 +5,6 @@ dm-haiku # mod_name=haiku flax protobuf pydriller -tqdm coverage scikit-learn # mod_name=sklearn pandas diff --git a/requirements/optional_gpu.txt b/requirements/optional_gpu.txt index 30b4c0dec9bcf..0810e836b9ba1 100644 --- a/requirements/optional_gpu.txt +++ b/requirements/optional_gpu.txt @@ -16,7 +16,6 @@ scipy # unpinned dm-haiku # unpinned mod_name=haiku flax pydriller -tqdm coverage scikit-learn # mod_name=sklearn pandas diff --git a/requirements/requirements.txt b/requirements/requirements.txt index a97b4537b56b0..375a11f7bbeaa 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -15,3 +15,4 @@ astunparse ml-dtypes # mod_name=ml_dtypes cloudpickle gast +tqdm