From c5c94558cac61b5e8d10a6ebf776e50520dfbb14 Mon Sep 17 00:00:00 2001 From: Magnus Holm Date: Fri, 21 Sep 2018 10:54:21 +0200 Subject: [PATCH 1/7] conda_mirror.py: abort download if diskspace below threshold --- conda_mirror/conda_mirror.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conda_mirror/conda_mirror.py b/conda_mirror/conda_mirror.py index 5be6126..41fcc49 100644 --- a/conda_mirror/conda_mirror.py +++ b/conda_mirror/conda_mirror.py @@ -26,6 +26,7 @@ 'win-64', 'win-32'] +MINIMUM_FREE_SPACE_MB = 100 def _maybe_split_channel(channel): """Split channel if it is fully qualified. @@ -678,6 +679,10 @@ def main(upstream_channel, target_directory, temp_directory, platform, with tempfile.TemporaryDirectory(dir=temp_directory) as download_dir: logger.info('downloading to the tempdir %s', download_dir) for package_name in sorted(to_mirror): + disk = os.statvfs(download_dir) + if ((disk.f_bavail * disk.f_frsize) / (1024 * 1024)) < MINIMUM_FREE_SPACE_MB: + logger.error('Disk space below threshold in %s. Aborting download.', download_dir) + break url = download_url.format( channel=channel, platform=platform, From a15049683d2a7120a8eb9fc53beb34c402196b1c Mon Sep 17 00:00:00 2001 From: Magnus Holm Date: Mon, 24 Sep 2018 13:31:15 +0200 Subject: [PATCH 2/7] Increased minimum_free_space_mb --- conda_mirror/conda_mirror.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_mirror/conda_mirror.py b/conda_mirror/conda_mirror.py index 41fcc49..6351dd8 100644 --- a/conda_mirror/conda_mirror.py +++ b/conda_mirror/conda_mirror.py @@ -26,7 +26,7 @@ 'win-64', 'win-32'] -MINIMUM_FREE_SPACE_MB = 100 +MINIMUM_FREE_SPACE_MB = 1000 def _maybe_split_channel(channel): """Split channel if it is fully qualified. From e92e97e17d5365a29930b66d862cf42584da28a6 Mon Sep 17 00:00:00 2001 From: Magnus Holm Date: Fri, 21 Sep 2018 10:55:44 +0200 Subject: [PATCH 3/7] conda_mirror.py: log download errors instead of breaking and clearing cache --- conda_mirror/conda_mirror.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/conda_mirror/conda_mirror.py b/conda_mirror/conda_mirror.py index 6351dd8..5623496 100644 --- a/conda_mirror/conda_mirror.py +++ b/conda_mirror/conda_mirror.py @@ -687,8 +687,12 @@ def main(upstream_channel, target_directory, temp_directory, platform, channel=channel, platform=platform, file_name=package_name) - _download(url, download_dir) - summary['downloaded'].add((url, download_dir)) + try: + _download(url, download_dir) + summary['downloaded'].add((url, download_dir)) + except: + logger.error('Unexpected error: %s. Aborting download.', sys.exc_info()[0]) + break # validate all packages in the download directory validation_results = _validate_packages(packages, download_dir, From c977191527d279fe839598d64c05a622f50aff1c Mon Sep 17 00:00:00 2001 From: Magnus Holm Date: Wed, 24 Oct 2018 18:05:44 +0200 Subject: [PATCH 4/7] conda_mirror.py: add minimum-free-space argument, improve space check --- conda_mirror/conda_mirror.py | 41 ++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/conda_mirror/conda_mirror.py b/conda_mirror/conda_mirror.py index d99b460..43a844c 100644 --- a/conda_mirror/conda_mirror.py +++ b/conda_mirror/conda_mirror.py @@ -26,7 +26,6 @@ 'win-64', 'win-32'] -MINIMUM_FREE_SPACE_MB = 1000 def _maybe_split_channel(channel): """Split channel if it is fully qualified. @@ -186,6 +185,12 @@ def _make_arg_parser(): help="Skip validation of files already present in target-directory", default=False, ) + ap.add_argument( + '--minimum-free-space', + help=("Threshold for free diskspace. Given in megabytes."), + type=int, + default=1000, + ) return ap @@ -260,6 +265,7 @@ def pdb_hook(exctype, value, traceback): 'whitelist': whitelist, 'dry_run': args.dry_run, 'no_validate_target': args.no_validate_target, + 'minimum_free_space': args.minimum_free_space, } @@ -518,7 +524,7 @@ def _validate_or_remove_package(args): def main(upstream_channel, target_directory, temp_directory, platform, blacklist=None, whitelist=None, num_threads=1, dry_run=False, - no_validate_target=False): + no_validate_target=False, minimum_free_space=0): """ Parameters @@ -558,6 +564,8 @@ def main(upstream_channel, target_directory, temp_directory, platform, no_validate_target : bool, optional Defaults to False. If True, skip validation of files already present in target_directory. + minimum_free_space : int, optional + Stop downloading when free space target_directory or temp_directory reach this threshold. Returns ------- @@ -690,24 +698,35 @@ def main(upstream_channel, target_directory, temp_directory, platform, # b. validate contents of temp file # c. move to local repo # mirror all new packages + minimum_free_space_kb = (minimum_free_space * 1024 * 1024) download_url, channel = _maybe_split_channel(upstream_channel) with tempfile.TemporaryDirectory(dir=temp_directory) as download_dir: logger.info('downloading to the tempdir %s', download_dir) for package_name in sorted(to_mirror): - disk = os.statvfs(download_dir) - if ((disk.f_bavail * disk.f_frsize) / (1024 * 1024)) < MINIMUM_FREE_SPACE_MB: - logger.error('Disk space below threshold in %s. Aborting download.', download_dir) - break url = download_url.format( channel=channel, platform=platform, file_name=package_name) try: - _download(url, download_dir) - summary['downloaded'].add((url, download_dir)) - except: - logger.error('Unexpected error: %s. Aborting download.', sys.exc_info()[0]) - break + # make sure we have enough free disk space in the temp folder to meet threshold + if shutil.disk_usage(download_dir).free < minimum_free_space_kb: + logger.error('Disk space below threshold in %s. Aborting download.', + download_dir) + break + + # download package + _download(url, download_dir) + + # make sure we have enough free disk space in the target folder to meet threshold + if shutil.disk_usage(local_directory).free < minimum_free_space_kb: + logger.error('Disk space below threshold in %s. Aborting download', + local_directory) + break + + summary['downloaded'].add((url, download_dir)) + except BaseException: + logger.error('Unexpected error: %s. Aborting download.', sys.exc_info()[0]) + break # validate all packages in the download directory validation_results = _validate_packages(packages, download_dir, From 9a281f8e5fdd8092cbf7ac2a6b3a219091d40dd5 Mon Sep 17 00:00:00 2001 From: Magnus Holm Date: Wed, 24 Oct 2018 18:07:08 +0200 Subject: [PATCH 5/7] conda_mirror.py: include already downloaded packages in space check --- conda_mirror/conda_mirror.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/conda_mirror/conda_mirror.py b/conda_mirror/conda_mirror.py index 43a844c..e51af18 100644 --- a/conda_mirror/conda_mirror.py +++ b/conda_mirror/conda_mirror.py @@ -387,7 +387,13 @@ def _download(url, target_directory): The url to download target_directory : str The path to a directory where `url` should be downloaded + + Returns + ------- + file_size: int + The size in bytes of the file that was downloaded """ + file_size = 0 chunk_size = 1024 # 1KB chunks logger.info("download_url=%s", url) # create a temporary file @@ -398,6 +404,8 @@ def _download(url, target_directory): ret = requests.get(url, stream=True) for data in ret.iter_content(chunk_size): tf.write(data) + file_size = os.path.getsize(download_filename) + return file_size def _list_conda_packages(local_dir): @@ -698,6 +706,7 @@ def main(upstream_channel, target_directory, temp_directory, platform, # b. validate contents of temp file # c. move to local repo # mirror all new packages + total_bytes = 0 minimum_free_space_kb = (minimum_free_space * 1024 * 1024) download_url, channel = _maybe_split_channel(upstream_channel) with tempfile.TemporaryDirectory(dir=temp_directory) as download_dir: @@ -715,10 +724,11 @@ def main(upstream_channel, target_directory, temp_directory, platform, break # download package - _download(url, download_dir) + total_bytes += _download(url, download_dir) # make sure we have enough free disk space in the target folder to meet threshold - if shutil.disk_usage(local_directory).free < minimum_free_space_kb: + # while also being able to fit the packages we have already downloaded + if (shutil.disk_usage(local_directory).free - total_bytes) < minimum_free_space_kb: logger.error('Disk space below threshold in %s. Aborting download', local_directory) break From 44a2ca41d662cd668754454742e67f8ff1e8a359 Mon Sep 17 00:00:00 2001 From: Magnus Holm Date: Thu, 25 Oct 2018 13:15:52 +0200 Subject: [PATCH 6/7] conda_mirror.py: updated exception handing in accordance to 58f2a38 --- conda_mirror/conda_mirror.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_mirror/conda_mirror.py b/conda_mirror/conda_mirror.py index e51af18..7b7bbcb 100644 --- a/conda_mirror/conda_mirror.py +++ b/conda_mirror/conda_mirror.py @@ -734,8 +734,8 @@ def main(upstream_channel, target_directory, temp_directory, platform, break summary['downloaded'].add((url, download_dir)) - except BaseException: - logger.error('Unexpected error: %s. Aborting download.', sys.exc_info()[0]) + except Exception as ex: + logger.error('Unexpected error: %s. Aborting download.', ex) break # validate all packages in the download directory From a29a55dad8b224a9305695292255f13c534e872f Mon Sep 17 00:00:00 2001 From: magnuhho <39699842+magnuhho@users.noreply.github.com> Date: Thu, 25 Oct 2018 16:09:57 +0200 Subject: [PATCH 7/7] conda_mirror.py: changed log facility back to exception --- conda_mirror/conda_mirror.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_mirror/conda_mirror.py b/conda_mirror/conda_mirror.py index d0b85ad..a330f65 100644 --- a/conda_mirror/conda_mirror.py +++ b/conda_mirror/conda_mirror.py @@ -735,7 +735,7 @@ def main(upstream_channel, target_directory, temp_directory, platform, summary['downloaded'].add((url, download_dir)) except Exception as ex: - logger.error('Unexpected error: %s. Aborting download.', ex) + logger.exception('Unexpected error: %s. Aborting download.', ex) break # validate all packages in the download directory