Skip to content

Commit

Permalink
Delete unused code
Browse files Browse the repository at this point in the history
  • Loading branch information
@ericdill
ericdill committed Feb 15, 2017
1 parent 98303f5 commit f0888f3
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 105 deletions.
152 changes: 68 additions & 84 deletions conda_mirror/conda_mirror.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,7 @@

DEFAULT_BAD_LICENSES = ['agpl', '']

DEFAULT_PLATFORMS = ['linux-64',
'linux-32',
'osx-64',
'win-64',
'win-32']
DEFAULT_PLATFORMS = ['linux-64', 'linux-32', 'osx-64', 'win-64', 'win-32']


def _maybe_split_channel(channel):
Expand Down Expand Up @@ -63,7 +59,8 @@ def _maybe_split_channel(channel):
# looks like we are being given a fully qualified channel
download_base, channel = channel.rsplit('/', 1)
download_template = download_base + url_suffix
logger.debug('download_template=%s. channel=%s', download_template, channel)
logger.debug('download_template=%s. channel=%s', download_template,
channel)
return download_template, channel


Expand All @@ -84,9 +81,10 @@ def _match(all_packages, key_glob_dict):
(key, glob_value) tuples
"""
matched = dict()
key_glob_dict = {key.lower(): glob.lower()
for key, glob
in key_glob_dict.items()}
key_glob_dict = {
key.lower(): glob.lower()
for key, glob in key_glob_dict.items()
}
for pkg_name, pkg_info in all_packages.items():
matched_all = []
# normalize the strings so that comparisons are easier
Expand All @@ -111,66 +109,63 @@ def _make_arg_parser():
argument_parser : argparse.ArgumentParser
The instantiated argument parser for this CLI
"""
ap = argparse.ArgumentParser(description="CLI interface for conda-mirror.py")
ap = argparse.ArgumentParser(
description="CLI interface for conda-mirror.py")

ap.add_argument(
'--upstream-channel',
help=('The target channel to mirror. Can be a channel on anaconda.org '
'like "conda-forge" or a full qualified channel like '
'"https://repo.continuum.io/pkgs/free/"'),
)
'"https://repo.continuum.io/pkgs/free/"'), )
ap.add_argument(
'--target-directory',
help='The place where packages should be mirrored to',
)
help='The place where packages should be mirrored to', )
ap.add_argument(
'--temp-directory',
help=('Temporary download location for the packages. Defaults to a '
'randomly selected temporary directory. Note that you might need '
'to specify a different location if your default temp directory '
'has less available space than your mirroring target'),
default=tempfile.gettempdir()
)
help=(
'Temporary download location for the packages. Defaults to a '
'randomly selected temporary directory. Note that you might need '
'to specify a different location if your default temp directory '
'has less available space than your mirroring target'),
default=tempfile.gettempdir())
ap.add_argument(
'--platform',
help=("The OS platform(s) to mirror. one of: {'linux-64', 'linux-32',"
"'osx-64', 'win-32', 'win-64'}"),
)
"'osx-64', 'win-32', 'win-64'}"), )
ap.add_argument(
'-v', '--verbose',
'-v',
'--verbose',
action="count",
help=("logging defaults to error/exception only. Takes up to three "
"'-v' flags. '-v': warning. '-vv': info. '-vvv': debug."),
default=0,
)
default=0, )
ap.add_argument(
'--config',
action="store",
help="Path to the yaml config file",
)
help="Path to the yaml config file", )
ap.add_argument(
'--pdb',
action="store_true",
help="Enable PDB debugging on exception",
default=False,
)
default=False, )
ap.add_argument(
'--version',
action="store_true",
help="Print version and quit",
default=False,
)
default=False, )
return ap


def _init_logger(verbosity):
# set up the logger
global logger
logger = logging.getLogger('conda_mirror')
logmap = {0: logging.ERROR,
1: logging.WARNING,
2: logging.INFO,
3: logging.DEBUG}
logmap = {
0: logging.ERROR,
1: logging.WARNING,
2: logging.INFO,
3: logging.DEBUG
}
loglevel = logmap.get(verbosity, '3')

# clear all handlers
Expand All @@ -185,8 +180,9 @@ def _init_logger(verbosity):

logger.addHandler(stream_handler)

print("Log level set to %s" % logging.getLevelName(logmap[verbosity]),
file=sys.stdout)
print(
"Log level set to %s" % logging.getLevelName(logmap[verbosity]),
file=sys.stdout)


def cli():
Expand All @@ -212,6 +208,7 @@ def cli():
# set the pdb_hook as the except hook for all exceptions
def pdb_hook(exctype, value, traceback):
pdb.post_mortem(traceback)

sys.excepthook = pdb_hook

config_dict = {}
Expand Down Expand Up @@ -241,7 +238,6 @@ def _remove_package(pkg_path, reason):
os.remove(pkg_path)



def _validate(filename, md5=None, sha256=None, size=None):
"""Validate the conda package tarfile located at `filename` with any of the
passed in options `md5`, `sha256` or `size. Also implicitly validate that
Expand All @@ -265,8 +261,9 @@ def _validate(filename, md5=None, sha256=None, size=None):
t = tarfile.open(filename)
t.extractfile('info/index.json').read().decode('utf-8')
except tarfile.TarError:
logger.info("Validation failed because conda package is corrupted.",
exc_info=True)
logger.info(
"Validation failed because conda package is corrupted.",
exc_info=True)
_remove_package(filename, reason="Tarfile read failure")
return
if size:
Expand All @@ -277,13 +274,15 @@ def _validate(filename, md5=None, sha256=None, size=None):
if calc != md5:
_remove_package(
filename,
reason="Failed md5 validation. Expected: %s. Computed: %s" % (calc, md5))
reason="Failed md5 validation. Expected: %s. Computed: %s" %
(calc, md5))
if sha256:
calc = hashlib.sha256(open(filename, 'rb').read()).hexdigest()
if calc != md5:
_remove_package(
filename,
reason="Failed sha256 validation. Expected: %s. Computed: %s" % (calc, sha256))
reason="Failed sha256 validation. Expected: %s. Computed: %s" %
(calc, sha256))


def get_repodata(channel, platform):
Expand All @@ -303,8 +302,8 @@ def get_repodata(channel, platform):
keyed on package name (e.g., twisted-16.0.0-py35_0.tar.bz2)
"""
url_template, channel = _maybe_split_channel(channel)
url = url_template.format(channel=channel, platform=platform,
file_name='repodata.json')
url = url_template.format(
channel=channel, platform=platform, file_name='repodata.json')
json = requests.get(url).json()
return json.get('info', {}), json.get('packages', {})

Expand Down Expand Up @@ -373,40 +372,27 @@ def _validate_packages(package_repodata, package_directory):
except KeyError:
logger.warning("%s is not in the upstream index. Removing...",
package)
_remove_package(os.path.join(package_directory, package),
reason="Package is not in the repodata index")
_remove_package(
os.path.join(package_directory, package),
reason="Package is not in the repodata index")
else:
# validate the integrity of the package, the size of the package and
# its hashes
logger.info('Validating %s. %s of %s', package, idx,
len(local_packages))
_validate(os.path.join(package_directory, package),
md5=package_metadata.get('md5'),
sha256=package_metadata.get('sha256'),
size=package_metadata.get('size'))


def _remove_local_blacklisted(blacklist, local_dir):
"""Removes any local conda packages that are blacklisted.
Parameters
----------
blacklist : list
Packages that should not be in `local_dir`
local_dir : str
Local directory to check for blacklisted conda packages
"""
# get list of current packages in folder
local_packages = _list_conda_packages(local_dir)
# if any are not in the final mirror list, remove them
for package_name in local_packages:
if package_name in blacklist:
_remove_package(os.path.join(local_dir, package_name),
reason="Package is blacklisted")


def main(upstream_channel, target_directory, temp_directory, platform,
blacklist=None, whitelist=None):
_validate(
os.path.join(package_directory, package),
md5=package_metadata.get('md5'),
sha256=package_metadata.get('sha256'),
size=package_metadata.get('size'))


def main(upstream_channel,
target_directory,
temp_directory,
platform,
blacklist=None,
whitelist=None):
"""
Parameters
Expand Down Expand Up @@ -538,15 +524,12 @@ def main(upstream_channel, target_directory, temp_directory, platform,
logger.info('downloading to the tempdir %s', download_dir)
for package_name in sorted(to_mirror):
url = download_url.format(
channel=channel,
platform=platform,
file_name=package_name)
channel=channel, platform=platform, file_name=package_name)
_download(url, download_dir)

# validate all packages in the download directory
_validate_packages(packages, download_dir)
logger.debug('contents of %s are %s',
download_dir,
logger.debug('contents of %s are %s', download_dir,
pformat(os.listdir(download_dir)))

# 8. Use already downloaded repodata.json contents but prune it of
Expand All @@ -555,13 +538,15 @@ def main(upstream_channel, target_directory, temp_directory, platform,

repodata = {'info': info, 'packages': packages}
# compute the packages that we have locally
packages_we_have = set(local_packages +
_list_conda_packages(download_dir))
packages_we_have = set(local_packages + _list_conda_packages(
download_dir))
# remake the packages dictionary with only the packages we have
# locally
repodata['packages'] = {
name: info for name, info in repodata['packages'].items()
if name in packages_we_have}
name: info
for name, info in repodata['packages'].items()
if name in packages_we_have
}
_write_repodata(download_dir, repodata)

# move new conda packages
Expand Down Expand Up @@ -592,8 +577,7 @@ def _write_repodata(package_dir, repodata_dict):
if not data.endswith('\n'):
data += '\n'

with open(os.path.join(package_dir,
'repodata.json'), 'w') as fo:
with open(os.path.join(package_dir, 'repodata.json'), 'w') as fo:
fo.write(data)

# compress repodata.json into the bz2 format. some conda commands still
Expand Down
21 changes: 0 additions & 21 deletions test/test_conda_mirror.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,24 +120,3 @@ def test_handling_bad_package(tmpdir, repodata):
conda_mirror._validate_packages(anaconda_repodata, bad_pkg_root)
assert bad_pkg_name not in os.listdir(bad_pkg_root)


def test_local_blacklisted_package(tmpdir):
local_repo_root = tmpdir.mkdir('repo').strpath
pkg_root = os.path.join(local_repo_root, 'linux-64')
os.makedirs(pkg_root)
blacklisted_pkg_name = 'remove-1-0.tar.bz2'
non_blacklisted_pkg_name = 'keep-1-0.tar.bz2'
with bz2.BZ2File(os.path.join(pkg_root, blacklisted_pkg_name), 'wb') as f:
f.write("This is a blacklisted package".encode())
with bz2.BZ2File(os.path.join(pkg_root, non_blacklisted_pkg_name), 'wb') as f:
f.write("This is not a blacklisted package".encode())
blacklist = [blacklisted_pkg_name]

# Test removal of local blacklisted packages
conda_mirror.logger.info("Testing %s", blacklisted_pkg_name)
assert blacklisted_pkg_name in os.listdir(pkg_root)
conda_mirror._remove_local_blacklisted(blacklist, pkg_root)
assert blacklisted_pkg_name not in os.listdir(pkg_root)
assert non_blacklisted_pkg_name in os.listdir(pkg_root)


0 comments on commit f0888f3

Please sign in to comment.