From 4756026b181f23ec0481cf372dd02118e559c2c8 Mon Sep 17 00:00:00 2001 From: Victor Quach Date: Wed, 6 Mar 2019 14:32:33 -0500 Subject: [PATCH 01/14] Pass kwargs to tqdm --- p_tqdm/__init__.py | 10 +++++----- p_tqdm/tests/tests.py | 10 ++++++++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/p_tqdm/__init__.py b/p_tqdm/__init__.py index d3308dd..3f57a39 100644 --- a/p_tqdm/__init__.py +++ b/p_tqdm/__init__.py @@ -43,8 +43,8 @@ def _parallel(ordered, function, *arrays, **kwargs): arrays = list(arrays) # Extract kwargs - num_cpus = kwargs.get('num_cpus', None) - num_iter = kwargs.get('num_iter', 1) + num_cpus = kwargs.pop('num_cpus', None) + num_iter = kwargs.pop('num_iter', 1) # Determine num_cpus if num_cpus is None: @@ -67,7 +67,7 @@ def _parallel(ordered, function, *arrays, **kwargs): # Create parallel iterator map_type = 'imap' if ordered else 'uimap' iterator = tqdm(getattr(Pool(num_cpus), map_type)(function, *arrays), - total=num_iter) + total=num_iter, **kwargs) return iterator @@ -131,7 +131,7 @@ def _sequential(function, *arrays, **kwargs): arrays = list(arrays) # Extract kwargs - num_iter = kwargs.get('num_iter', 1) + num_iter = kwargs.pop('num_iter', 1) # Determine num_iter when at least one list is present if any([type(array) == list for array in arrays]): @@ -147,7 +147,7 @@ def _sequential(function, *arrays, **kwargs): # Create parallel iterator iterator = tqdm(map(function, *arrays), - total=num_iter) + total=num_iter, **kwargs) return iterator diff --git a/p_tqdm/tests/tests.py b/p_tqdm/tests/tests.py index 0d95c95..4114c32 100644 --- a/p_tqdm/tests/tests.py +++ b/p_tqdm/tests/tests.py @@ -11,9 +11,9 @@ def add_2(a, b): def add_3(a, b, c): return a + b + c -def _test_one_list(self): +def _test_one_list(self, **kwargs): array = [1, 2, 3] - result = self.func(add_1, array) + result = self.func(add_1, array, **kwargs) if self.generator: result = list(result) @@ -302,5 +302,11 @@ def test_two_singles(self): def test_two_singles_with_num_iter(self): _test_two_singles_with_num_iter(self) + def test_dynamic_ncols(self): + _test_one_list(self, dynamic_ncols=True) + + def test_fixed_ncols(self): + _test_one_list(self, ncols=80) + if __name__ == '__main__': unittest.main() From 1d25945eb8d7717656898986ae847a60534c0879 Mon Sep 17 00:00:00 2001 From: Mike Clark Date: Wed, 27 Nov 2019 12:32:00 +0000 Subject: [PATCH 02/14] support for jupyter notebook widgets --- p_tqdm/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p_tqdm/__init__.py b/p_tqdm/__init__.py index d3308dd..3b731e0 100644 --- a/p_tqdm/__init__.py +++ b/p_tqdm/__init__.py @@ -10,7 +10,7 @@ from pathos.helpers import cpu_count from pathos.multiprocessing import ProcessingPool as Pool -from tqdm import tqdm +from tqdm.auto import tqdm def _parallel(ordered, function, *arrays, **kwargs): """Returns an iterator for a parallel map with a progress bar. From 658be1e04cea09379dbf3e2852a6ed0c9d5264f6 Mon Sep 17 00:00:00 2001 From: Kyle Swanson Date: Sat, 28 Dec 2019 00:45:10 +0000 Subject: [PATCH 03/14] Refactoring and restructuring code and tests --- .gitignore | 1 + LICENSE.txt | 2 +- p_tqdm/__init__.py | 177 ++-------------------- p_tqdm/p_tqdm.py | 175 ++++++++++++++++++++++ p_tqdm/tests/__init__.py | 0 p_tqdm/tests/tests.py | 306 --------------------------------------- requirements.txt | 2 - setup.py | 42 ++++-- tests/tests.py | 172 ++++++++++++++++++++++ 9 files changed, 389 insertions(+), 488 deletions(-) create mode 100644 p_tqdm/p_tqdm.py delete mode 100644 p_tqdm/tests/__init__.py delete mode 100644 p_tqdm/tests/tests.py delete mode 100644 requirements.txt create mode 100644 tests/tests.py diff --git a/.gitignore b/.gitignore index 090ac28..f966d33 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ gifs p_tqdm.egg-info __pycache__ *.pyc +.idea diff --git a/LICENSE.txt b/LICENSE.txt index cab87d4..b2f35b8 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2017 Kyle Swanson +Copyright (c) 2019 Kyle Swanson Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/p_tqdm/__init__.py b/p_tqdm/__init__.py index d3308dd..06be119 100644 --- a/p_tqdm/__init__.py +++ b/p_tqdm/__init__.py @@ -1,167 +1,10 @@ -"""Map functions with tqdm progress bars for parallel and sequential processing. - -p_map: Performs a parallel ordered map. -p_imap: Returns an iterator for a parallel ordered map. -p_umap: Performs a parallel unordered map. -p_uimap: Returns an iterator for a parallel unordered map. -t_map: Performs a sequential map. -t_imap: Returns an iterator for a sequential map. -""" - -from pathos.helpers import cpu_count -from pathos.multiprocessing import ProcessingPool as Pool -from tqdm import tqdm - -def _parallel(ordered, function, *arrays, **kwargs): - """Returns an iterator for a parallel map with a progress bar. - - Arguments: - ordered(bool): True for an ordered map, false for an unordered map. - function(function): The function to apply to each element - of the given arrays. - arrays(tuple): One or more arrays of the same length - containing the data to be mapped. If a non-list - variable is passed, it will be repeated a number - of times equal to the lengths of the list(s). If only - non-list variables are passed, the function will be - performed num_iter times. - num_cpus(int): The number of cpus to use in parallel. - If an int, uses that many cpus. - If a float, uses that proportion of cpus. - If None, uses all available cpus. - num_iter(int): If only non-list variables are passed, the - function will be performed num_iter times on - these variables. Default: 1. - - Returns: - An iterator which will apply the function - to each element of the given arrays in - parallel in order with a progress bar. - """ - - # Convert tuple to list - arrays = list(arrays) - - # Extract kwargs - num_cpus = kwargs.get('num_cpus', None) - num_iter = kwargs.get('num_iter', 1) - - # Determine num_cpus - if num_cpus is None: - num_cpus = cpu_count() - elif type(num_cpus) == float: - num_cpus = int(round(num_cpus * cpu_count())) - - # Determine num_iter when at least one list is present - if any([type(array) == list for array in arrays]): - num_iter = max([len(array) for array in arrays if type(array) == list]) - - # Convert single variables to lists - # and confirm lists are same length - for i, array in enumerate(arrays): - if type(array) != list: - arrays[i] = [array for _ in range(num_iter)] - else: - assert len(array) == num_iter - - # Create parallel iterator - map_type = 'imap' if ordered else 'uimap' - iterator = tqdm(getattr(Pool(num_cpus), map_type)(function, *arrays), - total=num_iter) - - return iterator - -def p_map(function, *arrays, **kwargs): - """Performs a parallel ordered map with a progress bar.""" - - ordered = True - iterator = _parallel(ordered, function, *arrays, **kwargs) - result = list(iterator) - - return result - -def p_imap(function, *arrays, **kwargs): - """Returns an iterator for a parallel ordered map with a progress bar.""" - - ordered = True - iterator = _parallel(ordered, function, *arrays, **kwargs) - - return iterator - -def p_umap(function, *arrays, **kwargs): - """Performs a parallel unordered map with a progress bar.""" - - ordered = False - iterator = _parallel(ordered, function, *arrays, **kwargs) - result = list(iterator) - - return result - -def p_uimap(function, *arrays, **kwargs): - """Returns an iterator for a parallel unordered map with a progress bar.""" - - ordered = False - iterator = _parallel(ordered, function, *arrays, **kwargs) - - return iterator - -def _sequential(function, *arrays, **kwargs): - """Returns an iterator for a sequential map with a progress bar. - - Arguments: - function(function): The function to apply to each element - of the given arrays. - arrays(tuple): One or more arrays of the same length - containing the data to be mapped. If a non-list - variable is passed, it will be repeated a number - of times equal to the lengths of the list(s). If only - non-list variables are passed, the function will be - performed num_iter times. - num_iter(int): If only non-list variables are passed, the - function will be performed num_iter times on - these variables. Default: 1. - - Returns: - An iterator which will apply the function - to each element of the given arrays sequentially - in order with a progress bar. - """ - - # Convert tuple to list - arrays = list(arrays) - - # Extract kwargs - num_iter = kwargs.get('num_iter', 1) - - # Determine num_iter when at least one list is present - if any([type(array) == list for array in arrays]): - num_iter = max([len(array) for array in arrays if type(array) == list]) - - # Convert single variables to lists - # and confirm lists are same length - for i, array in enumerate(arrays): - if type(array) != list: - arrays[i] = [array for _ in range(num_iter)] - else: - assert len(array) == num_iter - - # Create parallel iterator - iterator = tqdm(map(function, *arrays), - total=num_iter) - - return iterator - -def t_map(function, *arrays, **kwargs): - """Performs a sequential map with a progress bar.""" - - iterator = _sequential(function, *arrays, **kwargs) - result = list(iterator) - - return result - -def t_imap(function, *arrays, **kwargs): - """Returns an iterator for a sequential map with a progress bar.""" - - iterator = _sequential(function, *arrays, **kwargs) - - return iterator +from p_tqdm.p_tqdm import p_map, p_imap, p_umap, p_uimap, t_map, t_imap + +__all__ = [ + 'p_map', + 'p_imap', + 'p_umap', + 'p_uimap', + 't_map', + 't_imap' +] diff --git a/p_tqdm/p_tqdm.py b/p_tqdm/p_tqdm.py new file mode 100644 index 0000000..2427662 --- /dev/null +++ b/p_tqdm/p_tqdm.py @@ -0,0 +1,175 @@ +"""Map functions with tqdm progress bars for parallel and sequential processing. + +p_map: Performs a parallel ordered map. +p_imap: Returns an iterator for a parallel ordered map. +p_umap: Performs a parallel unordered map. +p_uimap: Returns an iterator for a parallel unordered map. +t_map: Performs a sequential map. +t_imap: Returns an iterator for a sequential map. +""" + +from pathos.helpers import cpu_count +from pathos.multiprocessing import ProcessingPool as Pool +from tqdm import tqdm + + +def _parallel(ordered, function, *arrays, **kwargs): + """Returns an iterator for a parallel map with a progress bar. + + Arguments: + ordered(bool): True for an ordered map, false for an unordered map. + function(function): The function to apply to each element + of the given arrays. + arrays(tuple): One or more arrays of the same length + containing the data to be mapped. If a non-list + variable is passed, it will be repeated a number + of times equal to the lengths of the list(s). If only + non-list variables are passed, the function will be + performed num_iter times. + num_cpus(int): The number of cpus to use in parallel. + If an int, uses that many cpus. + If a float, uses that proportion of cpus. + If None, uses all available cpus. + num_iter(int): If only non-list variables are passed, the + function will be performed num_iter times on + these variables. Default: 1. + + Returns: + An iterator which will apply the function + to each element of the given arrays in + parallel in order with a progress bar. + """ + + # Convert tuple to list + arrays = list(arrays) + + # Extract kwargs + num_cpus = kwargs.get('num_cpus', None) + num_iter = kwargs.get('num_iter', 1) + + # Determine num_cpus + if num_cpus is None: + num_cpus = cpu_count() + elif type(num_cpus) == float: + num_cpus = int(round(num_cpus * cpu_count())) + + # Determine num_iter when at least one list is present + if any([type(array) == list for array in arrays]): + num_iter = max([len(array) for array in arrays if type(array) == list]) + + # Convert single variables to lists + # and confirm lists are same length + for i, array in enumerate(arrays): + if type(array) != list: + arrays[i] = [array for _ in range(num_iter)] + else: + assert len(array) == num_iter + + # Create parallel iterator + map_type = 'imap' if ordered else 'uimap' + iterator = tqdm(getattr(Pool(num_cpus), map_type)(function, *arrays), + total=num_iter) + + return iterator + + +def p_map(function, *arrays, **kwargs): + """Performs a parallel ordered map with a progress bar.""" + + ordered = True + iterator = _parallel(ordered, function, *arrays, **kwargs) + result = list(iterator) + + return result + + +def p_imap(function, *arrays, **kwargs): + """Returns an iterator for a parallel ordered map with a progress bar.""" + + ordered = True + iterator = _parallel(ordered, function, *arrays, **kwargs) + + return iterator + + +def p_umap(function, *arrays, **kwargs): + """Performs a parallel unordered map with a progress bar.""" + + ordered = False + iterator = _parallel(ordered, function, *arrays, **kwargs) + result = list(iterator) + + return result + + +def p_uimap(function, *arrays, **kwargs): + """Returns an iterator for a parallel unordered map with a progress bar.""" + + ordered = False + iterator = _parallel(ordered, function, *arrays, **kwargs) + + return iterator + + +def _sequential(function, *arrays, **kwargs): + """Returns an iterator for a sequential map with a progress bar. + + Arguments: + function(function): The function to apply to each element + of the given arrays. + arrays(tuple): One or more arrays of the same length + containing the data to be mapped. If a non-list + variable is passed, it will be repeated a number + of times equal to the lengths of the list(s). If only + non-list variables are passed, the function will be + performed num_iter times. + num_iter(int): If only non-list variables are passed, the + function will be performed num_iter times on + these variables. Default: 1. + + Returns: + An iterator which will apply the function + to each element of the given arrays sequentially + in order with a progress bar. + """ + + # Convert tuple to list + arrays = list(arrays) + + # Extract kwargs + num_iter = kwargs.get('num_iter', 1) + + # Determine num_iter when at least one list is present + if any([type(array) == list for array in arrays]): + num_iter = max([len(array) for array in arrays if type(array) == list]) + + # Convert single variables to lists + # and confirm lists are same length + for i, array in enumerate(arrays): + if type(array) != list: + arrays[i] = [array for _ in range(num_iter)] + else: + assert len(array) == num_iter + + # Create parallel iterator + iterator = tqdm(map(function, *arrays), + total=num_iter) + + return iterator + + +def t_map(function, *arrays, **kwargs): + """Performs a sequential map with a progress bar.""" + + iterator = _sequential(function, *arrays, **kwargs) + result = list(iterator) + + return result + + +def t_imap(function, *arrays, **kwargs): + """Returns an iterator for a sequential map with a progress bar.""" + + iterator = _sequential(function, *arrays, **kwargs) + + return iterator diff --git a/p_tqdm/tests/__init__.py b/p_tqdm/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/p_tqdm/tests/tests.py b/p_tqdm/tests/tests.py deleted file mode 100644 index 0d95c95..0000000 --- a/p_tqdm/tests/tests.py +++ /dev/null @@ -1,306 +0,0 @@ -import unittest - -import p_tqdm - -def add_1(a): - return a + 1 - -def add_2(a, b): - return a + b - -def add_3(a, b, c): - return a + b + c - -def _test_one_list(self): - array = [1, 2, 3] - result = self.func(add_1, array) - if self.generator: - result = list(result) - - correct_array = [2, 3, 4] - if self.ordered: - self.assertEquals(correct_array, result) - else: - self.assertEquals(sorted(correct_array), sorted(result)) - -def _test_two_lists(self): - array_1 = [1, 2, 3] - array_2 = [10, 11, 12] - result = self.func(add_2, array_1, array_2) - if self.generator: - result = list(result) - - correct_array = [11, 13, 15] - if self.ordered: - self.assertEquals(correct_array, result) - else: - self.assertEquals(sorted(correct_array), sorted(result)) - -def _test_two_lists_and_one_single(self): - array_1 = [1, 2, 3] - array_2 = [10, 11, 12] - single = 5 - result = self.func(add_3, array_1, single, array_2) - if self.generator: - result = list(result) - - correct_array = [16, 18, 20] - if self.ordered: - self.assertEquals(correct_array, result) - else: - self.assertEquals(sorted(correct_array), sorted(result)) - -def _test_one_list_and_two_singles(self): - array = [1, 2, 3] - single_1 = 5 - single_2 = -2 - result = self.func(add_3, single_1, array, single_2) - if self.generator: - result = list(result) - - correct_array = [4, 5, 6] - if self.ordered: - self.assertEquals(correct_array, result) - else: - self.assertEquals(sorted(correct_array), sorted(result)) - -def _test_one_single(self): - single = 5 - result = self.func(add_1, single) - if self.generator: - result = list(result) - - correct_array = [6] - if self.ordered: - self.assertEquals(correct_array, result) - else: - self.assertEquals(sorted(correct_array), sorted(result)) - -def _test_one_single_with_num_iter(self): - single = 5 - num_iter = 3 - result = self.func(add_1, single, num_iter=num_iter) - if self.generator: - result = list(result) - - correct_array = [6]*num_iter - if self.ordered: - self.assertEquals(correct_array, result) - else: - self.assertEquals(sorted(correct_array), sorted(result)) - -def _test_two_singles(self): - single_1 = 5 - single_2 = -2 - result = self.func(add_2, single_1, single_2) - if self.generator: - result = list(result) - - correct_array = [3] - if self.ordered: - self.assertEquals(correct_array, result) - else: - self.assertEquals(sorted(correct_array), sorted(result)) - -def _test_two_singles_with_num_iter(self): - single_1 = 5 - single_2 = -2 - num_iter = 3 - result = self.func(add_2, single_1, single_2, num_iter=num_iter) - if self.generator: - result = list(result) - - correct_array = [3]*num_iter - if self.ordered: - self.assertEquals(correct_array, result) - else: - self.assertEquals(sorted(correct_array), sorted(result)) - -class Testp_imap(unittest.TestCase): - def __init__(self, *args, **kwargs): - super(Testp_imap, self).__init__(*args, **kwargs) - self.func = p_tqdm.p_imap - self.generator = True - self.ordered = True - - def test_one_list(self): - _test_one_list(self) - - def test_two_lists(self): - _test_two_lists(self) - - def test_two_lists_and_one_single(self): - _test_two_lists_and_one_single(self) - - def test_one_list_and_two_singles(self): - _test_one_list_and_two_singles(self) - - def test_one_single(self): - _test_one_single(self) - - def test_one_single_with_num_iter(self): - _test_one_single_with_num_iter(self) - - def test_two_singles(self): - _test_two_singles(self) - - def test_two_singles_with_num_iter(self): - _test_two_singles_with_num_iter(self) - -class Testp_map(unittest.TestCase): - def __init__(self, *args, **kwargs): - super(Testp_map, self).__init__(*args, **kwargs) - self.func = p_tqdm.p_map - self.generator = False - self.ordered = True - - def test_one_list(self): - _test_one_list(self) - - def test_two_lists(self): - _test_two_lists(self) - - def test_two_lists_and_one_single(self): - _test_two_lists_and_one_single(self) - - def test_one_list_and_two_singles(self): - _test_one_list_and_two_singles(self) - - def test_one_single(self): - _test_one_single(self) - - def test_one_single_with_num_iter(self): - _test_one_single_with_num_iter(self) - - def test_two_singles(self): - _test_two_singles(self) - - def test_two_singles_with_num_iter(self): - _test_two_singles_with_num_iter(self) - -class Testp_uimap(unittest.TestCase): - def __init__(self, *args, **kwargs): - super(Testp_uimap, self).__init__(*args, **kwargs) - self.func = p_tqdm.p_uimap - self.generator = True - self.ordered = False - - def test_one_list(self): - _test_one_list(self) - - def test_two_lists(self): - _test_two_lists(self) - - def test_two_lists_and_one_single(self): - _test_two_lists_and_one_single(self) - - def test_one_list_and_two_singles(self): - _test_one_list_and_two_singles(self) - - def test_one_single(self): - _test_one_single(self) - - def test_one_single_with_num_iter(self): - _test_one_single_with_num_iter(self) - - def test_two_singles(self): - _test_two_singles(self) - - def test_two_singles_with_num_iter(self): - _test_two_singles_with_num_iter(self) - -class Testp_umap(unittest.TestCase): - def __init__(self, *args, **kwargs): - super(Testp_umap, self).__init__(*args, **kwargs) - self.func = p_tqdm.p_umap - self.generator = False - self.ordered = False - - def test_one_list(self): - _test_one_list(self) - - def test_two_lists(self): - _test_two_lists(self) - - def test_two_lists_and_one_single(self): - _test_two_lists_and_one_single(self) - - def test_one_list_and_two_singles(self): - _test_one_list_and_two_singles(self) - - def test_one_single(self): - _test_one_single(self) - - def test_one_single_with_num_iter(self): - _test_one_single_with_num_iter(self) - - def test_two_singles(self): - _test_two_singles(self) - - def test_two_singles_with_num_iter(self): - _test_two_singles_with_num_iter(self) - -class Testt_imap(unittest.TestCase): - def __init__(self, *args, **kwargs): - super(Testt_imap, self).__init__(*args, **kwargs) - self.func = p_tqdm.t_imap - self.generator = True - self.ordered = True - - def test_one_list(self): - _test_one_list(self) - - def test_two_lists(self): - _test_two_lists(self) - - def test_two_lists_and_one_single(self): - _test_two_lists_and_one_single(self) - - def test_one_list_and_two_singles(self): - _test_one_list_and_two_singles(self) - - def test_one_single(self): - _test_one_single(self) - - def test_one_single_with_num_iter(self): - _test_one_single_with_num_iter(self) - - def test_two_singles(self): - _test_two_singles(self) - - def test_two_singles_with_num_iter(self): - _test_two_singles_with_num_iter(self) - -class Testt_map(unittest.TestCase): - def __init__(self, *args, **kwargs): - super(Testt_map, self).__init__(*args, **kwargs) - self.func = p_tqdm.t_map - self.generator = False - self.ordered = True - - def test_one_list(self): - _test_one_list(self) - - def test_two_lists(self): - _test_two_lists(self) - - def test_two_lists_and_one_single(self): - _test_two_lists_and_one_single(self) - - def test_one_list_and_two_singles(self): - _test_one_list_and_two_singles(self) - - def test_one_single(self): - _test_one_single(self) - - def test_one_single_with_num_iter(self): - _test_one_single_with_num_iter(self) - - def test_two_singles(self): - _test_two_singles(self) - - def test_two_singles_with_num_iter(self): - _test_two_singles_with_num_iter(self) - -if __name__ == '__main__': - unittest.main() diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index fcb1825..0000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -pathos -tqdm diff --git a/setup.py b/setup.py index cfa91c0..e8b75d6 100644 --- a/setup.py +++ b/setup.py @@ -1,17 +1,35 @@ -from setuptools import setup +from setuptools import find_packages, setup + +with open('README.md') as f: + long_description = f.read() setup( - name = 'p_tqdm', - packages = ['p_tqdm'], - version = '1.2', - description = 'Parallel processing with progress bars', - author = 'Kyle Swanson', - author_email = 'swansonk.14@gmail.com', - url = 'https://github.com/swansonk14/p_tqdm', - license = 'MIT', - install_requires = ['tqdm', 'pathos'], + name='p_tqdm', + version='1.2', + author='Kyle Swanson', + author_email='swansonk.14@gmail.com', + description='Parallel processing with progress bars', + long_description=long_description, + long_description_content_type='text/markdown', + url='https://github.com/swansonk14/p_tqdm', + download_url='https://github.com/swansonk14/p_tqdm/v_1.2.tar.gz', + license='MIT', + packages=find_packages(), + install_requires=[ + 'tqdm', + 'pathos' + ], test_suite='nose.collector', tests_require=['nose'], - keywords = ['tqdm', 'progress bar', 'parallel'], - classifiers = [], + classifiers=[ + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 3', + 'License :: OSI Approved :: MIT License', + 'Operating System :: OS Independent', + ], + keywords=[ + 'tqdm', + 'progress bar', + 'parallel' + ], ) diff --git a/tests/tests.py b/tests/tests.py new file mode 100644 index 0000000..11de08f --- /dev/null +++ b/tests/tests.py @@ -0,0 +1,172 @@ +import unittest + +from p_tqdm import p_map, p_imap, p_umap, p_uimap, t_map, t_imap + + +def add_1(a): + return a + 1 + + +def add_2(a, b): + return a + b + + +def add_3(a, b, c): + return a + b + c + + +class Test_p_map(unittest.TestCase): + def __init__(self, *args, **kwargs): + super(Test_p_map, self).__init__(*args, **kwargs) + self.func = p_map + self.generator = False + self.ordered = True + + def test_one_list(self): + array = [1, 2, 3] + result = self.func(add_1, array) + if self.generator: + result = list(result) + + correct_array = [2, 3, 4] + if self.ordered: + self.assertEquals(correct_array, result) + else: + self.assertEquals(sorted(correct_array), sorted(result)) + + def test_two_lists(self): + array_1 = [1, 2, 3] + array_2 = [10, 11, 12] + result = self.func(add_2, array_1, array_2) + if self.generator: + result = list(result) + + correct_array = [11, 13, 15] + if self.ordered: + self.assertEquals(correct_array, result) + else: + self.assertEquals(sorted(correct_array), sorted(result)) + + def test_two_lists_and_one_single(self): + array_1 = [1, 2, 3] + array_2 = [10, 11, 12] + single = 5 + result = self.func(add_3, array_1, single, array_2) + if self.generator: + result = list(result) + + correct_array = [16, 18, 20] + if self.ordered: + self.assertEquals(correct_array, result) + else: + self.assertEquals(sorted(correct_array), sorted(result)) + + def test_one_list_and_two_singles(self): + array = [1, 2, 3] + single_1 = 5 + single_2 = -2 + result = self.func(add_3, single_1, array, single_2) + if self.generator: + result = list(result) + + correct_array = [4, 5, 6] + if self.ordered: + self.assertEquals(correct_array, result) + else: + self.assertEquals(sorted(correct_array), sorted(result)) + + def test_one_single(self): + single = 5 + result = self.func(add_1, single) + if self.generator: + result = list(result) + + correct_array = [6] + if self.ordered: + self.assertEquals(correct_array, result) + else: + self.assertEquals(sorted(correct_array), sorted(result)) + + def test_one_single_with_num_iter(self): + single = 5 + num_iter = 3 + result = self.func(add_1, single, num_iter=num_iter) + if self.generator: + result = list(result) + + correct_array = [6]*num_iter + if self.ordered: + self.assertEquals(correct_array, result) + else: + self.assertEquals(sorted(correct_array), sorted(result)) + + def test_two_singles(self): + single_1 = 5 + single_2 = -2 + result = self.func(add_2, single_1, single_2) + if self.generator: + result = list(result) + + correct_array = [3] + if self.ordered: + self.assertEquals(correct_array, result) + else: + self.assertEquals(sorted(correct_array), sorted(result)) + + def test_two_singles_with_num_iter(self): + single_1 = 5 + single_2 = -2 + num_iter = 3 + result = self.func(add_2, single_1, single_2, num_iter=num_iter) + if self.generator: + result = list(result) + + correct_array = [3]*num_iter + if self.ordered: + self.assertEquals(correct_array, result) + else: + self.assertEquals(sorted(correct_array), sorted(result)) + + +class Test_p_imap(Test_p_map): + def __init__(self, *args, **kwargs): + super(Test_p_map, self).__init__(*args, **kwargs) + self.func = p_imap + self.generator = True + self.ordered = True + + +class Test_p_umap(Test_p_map): + def __init__(self, *args, **kwargs): + super(Test_p_map, self).__init__(*args, **kwargs) + self.func = p_umap + self.generator = False + self.ordered = False + + +class Test_p_uimap(Test_p_map): + def __init__(self, *args, **kwargs): + super(Test_p_map, self).__init__(*args, **kwargs) + self.func = p_uimap + self.generator = True + self.ordered = False + + +class Test_t_map(Test_p_map): + def __init__(self, *args, **kwargs): + super(Test_p_map, self).__init__(*args, **kwargs) + self.func = t_map + self.generator = False + self.ordered = True + + +class Test_t_imap(Test_p_map): + def __init__(self, *args, **kwargs): + super(Test_p_map, self).__init__(*args, **kwargs) + self.func = t_imap + self.generator = True + self.ordered = True + + +if __name__ == '__main__': + unittest.main() From ed5b2dec414f8658dcef602ba835b9dc5959e718 Mon Sep 17 00:00:00 2001 From: Kyle Swanson Date: Sat, 28 Dec 2019 01:16:35 +0000 Subject: [PATCH 04/14] Refactoring tests --- tests/tests.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 11de08f..7f063cc 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -30,9 +30,9 @@ def test_one_list(self): correct_array = [2, 3, 4] if self.ordered: - self.assertEquals(correct_array, result) + self.assertEqual(correct_array, result) else: - self.assertEquals(sorted(correct_array), sorted(result)) + self.assertEqual(sorted(correct_array), sorted(result)) def test_two_lists(self): array_1 = [1, 2, 3] @@ -43,9 +43,9 @@ def test_two_lists(self): correct_array = [11, 13, 15] if self.ordered: - self.assertEquals(correct_array, result) + self.assertEqual(correct_array, result) else: - self.assertEquals(sorted(correct_array), sorted(result)) + self.assertEqual(sorted(correct_array), sorted(result)) def test_two_lists_and_one_single(self): array_1 = [1, 2, 3] @@ -57,9 +57,9 @@ def test_two_lists_and_one_single(self): correct_array = [16, 18, 20] if self.ordered: - self.assertEquals(correct_array, result) + self.assertEqual(correct_array, result) else: - self.assertEquals(sorted(correct_array), sorted(result)) + self.assertEqual(sorted(correct_array), sorted(result)) def test_one_list_and_two_singles(self): array = [1, 2, 3] @@ -71,9 +71,9 @@ def test_one_list_and_two_singles(self): correct_array = [4, 5, 6] if self.ordered: - self.assertEquals(correct_array, result) + self.assertEqual(correct_array, result) else: - self.assertEquals(sorted(correct_array), sorted(result)) + self.assertEqual(sorted(correct_array), sorted(result)) def test_one_single(self): single = 5 @@ -83,9 +83,9 @@ def test_one_single(self): correct_array = [6] if self.ordered: - self.assertEquals(correct_array, result) + self.assertEqual(correct_array, result) else: - self.assertEquals(sorted(correct_array), sorted(result)) + self.assertEqual(sorted(correct_array), sorted(result)) def test_one_single_with_num_iter(self): single = 5 @@ -94,11 +94,11 @@ def test_one_single_with_num_iter(self): if self.generator: result = list(result) - correct_array = [6]*num_iter + correct_array = [6] * num_iter if self.ordered: - self.assertEquals(correct_array, result) + self.assertEqual(correct_array, result) else: - self.assertEquals(sorted(correct_array), sorted(result)) + self.assertEqual(sorted(correct_array), sorted(result)) def test_two_singles(self): single_1 = 5 @@ -109,9 +109,9 @@ def test_two_singles(self): correct_array = [3] if self.ordered: - self.assertEquals(correct_array, result) + self.assertEqual(correct_array, result) else: - self.assertEquals(sorted(correct_array), sorted(result)) + self.assertEqual(sorted(correct_array), sorted(result)) def test_two_singles_with_num_iter(self): single_1 = 5 @@ -121,11 +121,11 @@ def test_two_singles_with_num_iter(self): if self.generator: result = list(result) - correct_array = [3]*num_iter + correct_array = [3] * num_iter if self.ordered: - self.assertEquals(correct_array, result) + self.assertEqual(correct_array, result) else: - self.assertEquals(sorted(correct_array), sorted(result)) + self.assertEqual(sorted(correct_array), sorted(result)) class Test_p_imap(Test_p_map): From f2cc0767442a59532f2a3ec85d7c04e0abf9cc54 Mon Sep 17 00:00:00 2001 From: Kyle Swanson Date: Sat, 28 Dec 2019 01:16:48 +0000 Subject: [PATCH 05/14] Fix to prevent pool from hanging --- p_tqdm/p_tqdm.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/p_tqdm/p_tqdm.py b/p_tqdm/p_tqdm.py index 2427662..6db196f 100644 --- a/p_tqdm/p_tqdm.py +++ b/p_tqdm/p_tqdm.py @@ -9,7 +9,7 @@ """ from pathos.helpers import cpu_count -from pathos.multiprocessing import ProcessingPool as Pool +from pathos.multiprocessing import ProcessPool as Pool from tqdm import tqdm @@ -67,10 +67,13 @@ def _parallel(ordered, function, *arrays, **kwargs): # Create parallel iterator map_type = 'imap' if ordered else 'uimap' - iterator = tqdm(getattr(Pool(num_cpus), map_type)(function, *arrays), - total=num_iter) + pool = Pool(num_cpus) + map_func = getattr(pool, map_type) - return iterator + for item in tqdm(map_func(function, *arrays), total=num_iter): + yield item + + pool.clear() def p_map(function, *arrays, **kwargs): From 5042f2c70a79eddc93f44b6408cdde53b1468852 Mon Sep 17 00:00:00 2001 From: Kyle Swanson Date: Sat, 28 Dec 2019 01:31:27 +0000 Subject: [PATCH 06/14] Update travis tests --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5506405..3d8c077 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,9 @@ python: - "3.4" - "3.5" - "3.6" + - "3.7" + - "3.8" install: - - pip install -r requirements.txt + - pip install -e . script: - nosetests From 1e207d9f7cb6cd708aa2c202c36225f2de1215d3 Mon Sep 17 00:00:00 2001 From: Kyle Swanson Date: Sat, 28 Dec 2019 01:57:23 +0000 Subject: [PATCH 07/14] Add python version badges to readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 574f68b..c5ffb3b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # p_tqdm +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/p_tqdm)](https://badge.fury.io/py/p_tqdm) +[![PyPI version](https://badge.fury.io/py/p_tqdm.svg)](https://badge.fury.io/py/p_tqdm) [![Build Status](https://travis-ci.org/swansonk14/p_tqdm.svg?branch=master)](https://travis-ci.org/swansonk14/p_tqdm) `p_tqdm` makes parallel processing with progress bars easy. @@ -10,8 +12,6 @@ ```pip install p_tqdm``` -`p_tqdm` works with Python versions 2.7, 3.4, 3.5, 3.6. - ## Example Let's say you want to add two lists element by element. Without any parallelism, this can be done easily with a Python `map`. From 776196a79deed0651f8a950421b23457deb5a61f Mon Sep 17 00:00:00 2001 From: Kyle Swanson Date: Sat, 28 Dec 2019 01:57:42 +0000 Subject: [PATCH 08/14] Add typing and change support to Python 3.5+ --- .travis.yml | 2 -- p_tqdm/p_tqdm.py | 49 ++++++++++++++++++++++++------------------------ 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3d8c077..40f921c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,5 @@ language: python python: - - "2.7" - - "3.4" - "3.5" - "3.6" - "3.7" diff --git a/p_tqdm/p_tqdm.py b/p_tqdm/p_tqdm.py index 11ef73e..3e44dd5 100644 --- a/p_tqdm/p_tqdm.py +++ b/p_tqdm/p_tqdm.py @@ -8,19 +8,21 @@ t_imap: Returns an iterator for a sequential map. """ +from typing import Any, Callable, Generator, Optional + from pathos.helpers import cpu_count from pathos.multiprocessing import ProcessPool as Pool from tqdm.auto import tqdm -def _parallel(ordered, function, *arrays, **kwargs): - """Returns an iterator for a parallel map with a progress bar. +def _parallel(ordered: bool, function: Callable, *arrays: list, **kwargs: Any) -> Generator: + """Returns a generator for a parallel map with a progress bar. Arguments: ordered(bool): True for an ordered map, false for an unordered map. - function(function): The function to apply to each element + function(Callable): The function to apply to each element of the given arrays. - arrays(tuple): One or more arrays of the same length + arrays(Tuple[list]): One or more arrays of the same length containing the data to be mapped. If a non-list variable is passed, it will be repeated a number of times equal to the lengths of the list(s). If only @@ -35,7 +37,7 @@ def _parallel(ordered, function, *arrays, **kwargs): these variables. Default: 1. Returns: - An iterator which will apply the function + A generator which will apply the function to each element of the given arrays in parallel in order with a progress bar. """ @@ -44,8 +46,8 @@ def _parallel(ordered, function, *arrays, **kwargs): arrays = list(arrays) # Extract kwargs - num_cpus = kwargs.pop('num_cpus', None) - num_iter = kwargs.pop('num_iter', 1) + num_cpus: Optional[int] = kwargs.pop('num_cpus', None) + num_iter: int = kwargs.pop('num_iter', 1) # Determine num_cpus if num_cpus is None: @@ -65,7 +67,7 @@ def _parallel(ordered, function, *arrays, **kwargs): else: assert len(array) == num_iter - # Create parallel iterator + # Create parallel generator map_type = 'imap' if ordered else 'uimap' pool = Pool(num_cpus) map_func = getattr(pool, map_type) @@ -76,7 +78,7 @@ def _parallel(ordered, function, *arrays, **kwargs): pool.clear() -def p_map(function, *arrays, **kwargs): +def p_map(function: Callable, *arrays: list, **kwargs: Any) -> list: """Performs a parallel ordered map with a progress bar.""" ordered = True @@ -86,7 +88,7 @@ def p_map(function, *arrays, **kwargs): return result -def p_imap(function, *arrays, **kwargs): +def p_imap(function: Callable, *arrays: list, **kwargs: Any) -> Generator: """Returns an iterator for a parallel ordered map with a progress bar.""" ordered = True @@ -95,7 +97,7 @@ def p_imap(function, *arrays, **kwargs): return iterator -def p_umap(function, *arrays, **kwargs): +def p_umap(function: Callable, *arrays: list, **kwargs: Any) -> list: """Performs a parallel unordered map with a progress bar.""" ordered = False @@ -105,7 +107,7 @@ def p_umap(function, *arrays, **kwargs): return result -def p_uimap(function, *arrays, **kwargs): +def p_uimap(function: Callable, *arrays: list, **kwargs: Any) -> Generator: """Returns an iterator for a parallel unordered map with a progress bar.""" ordered = False @@ -114,13 +116,13 @@ def p_uimap(function, *arrays, **kwargs): return iterator -def _sequential(function, *arrays, **kwargs): - """Returns an iterator for a sequential map with a progress bar. +def _sequential(function: Callable, *arrays: list, **kwargs: Any) -> Generator: + """Returns a generator for a sequential map with a progress bar. Arguments: - function(function): The function to apply to each element + function(Callable): The function to apply to each element of the given arrays. - arrays(tuple): One or more arrays of the same length + arrays(Tuple[list]): One or more arrays of the same length containing the data to be mapped. If a non-list variable is passed, it will be repeated a number of times equal to the lengths of the list(s). If only @@ -131,7 +133,7 @@ def _sequential(function, *arrays, **kwargs): these variables. Default: 1. Returns: - An iterator which will apply the function + A generator which will apply the function to each element of the given arrays sequentially in order with a progress bar. """ @@ -140,7 +142,7 @@ def _sequential(function, *arrays, **kwargs): arrays = list(arrays) # Extract kwargs - num_iter = kwargs.pop('num_iter', 1) + num_iter: int = kwargs.pop('num_iter', 1) # Determine num_iter when at least one list is present if any([type(array) == list for array in arrays]): @@ -154,13 +156,12 @@ def _sequential(function, *arrays, **kwargs): else: assert len(array) == num_iter - # Create parallel iterator - iterator = tqdm(map(function, *arrays), total=num_iter, **kwargs) - - return iterator + # Create sequential generator + for item in tqdm(map(function, *arrays), total=num_iter, **kwargs): + yield item -def t_map(function, *arrays, **kwargs): +def t_map(function: Callable, *arrays: list, **kwargs: Any) -> list: """Performs a sequential map with a progress bar.""" iterator = _sequential(function, *arrays, **kwargs) @@ -169,7 +170,7 @@ def t_map(function, *arrays, **kwargs): return result -def t_imap(function, *arrays, **kwargs): +def t_imap(function: Callable, *arrays: list, **kwargs: Any) -> Generator: """Returns an iterator for a sequential map with a progress bar.""" iterator = _sequential(function, *arrays, **kwargs) From c16df32825fcb2aafb5a8bbcaaf9e76c1b14018d Mon Sep 17 00:00:00 2001 From: Kyle Swanson Date: Sat, 28 Dec 2019 02:00:25 +0000 Subject: [PATCH 09/14] Removing unnecessary type annotations to preserve compatibility with python 3.5 --- p_tqdm/p_tqdm.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/p_tqdm/p_tqdm.py b/p_tqdm/p_tqdm.py index 3e44dd5..7109232 100644 --- a/p_tqdm/p_tqdm.py +++ b/p_tqdm/p_tqdm.py @@ -8,7 +8,7 @@ t_imap: Returns an iterator for a sequential map. """ -from typing import Any, Callable, Generator, Optional +from typing import Any, Callable, Generator from pathos.helpers import cpu_count from pathos.multiprocessing import ProcessPool as Pool @@ -46,8 +46,8 @@ def _parallel(ordered: bool, function: Callable, *arrays: list, **kwargs: Any) - arrays = list(arrays) # Extract kwargs - num_cpus: Optional[int] = kwargs.pop('num_cpus', None) - num_iter: int = kwargs.pop('num_iter', 1) + num_cpus = kwargs.pop('num_cpus', None) + num_iter = kwargs.pop('num_iter', 1) # Determine num_cpus if num_cpus is None: @@ -142,7 +142,7 @@ def _sequential(function: Callable, *arrays: list, **kwargs: Any) -> Generator: arrays = list(arrays) # Extract kwargs - num_iter: int = kwargs.pop('num_iter', 1) + num_iter = kwargs.pop('num_iter', 1) # Determine num_iter when at least one list is present if any([type(array) == list for array in arrays]): From 656c7f2e6730ca40047c7a4364d434a45ca03c50 Mon Sep 17 00:00:00 2001 From: Kyle Swanson Date: Sat, 28 Dec 2019 02:02:56 +0000 Subject: [PATCH 10/14] Updating version to 1.3 --- setup.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index e8b75d6..69793ca 100644 --- a/setup.py +++ b/setup.py @@ -5,14 +5,14 @@ setup( name='p_tqdm', - version='1.2', + version='1.3', author='Kyle Swanson', author_email='swansonk.14@gmail.com', description='Parallel processing with progress bars', long_description=long_description, long_description_content_type='text/markdown', url='https://github.com/swansonk14/p_tqdm', - download_url='https://github.com/swansonk14/p_tqdm/v_1.2.tar.gz', + download_url='https://github.com/swansonk14/p_tqdm/v_1.3.tar.gz', license='MIT', packages=find_packages(), install_requires=[ @@ -22,8 +22,11 @@ test_suite='nose.collector', tests_require=['nose'], classifiers=[ - 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', ], From a451e5547deccfd7062cee67309ca08bb1847651 Mon Sep 17 00:00:00 2001 From: Kyle Swanson Date: Sun, 29 Dec 2019 00:59:54 +0000 Subject: [PATCH 11/14] Fixing pypi version badge in readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c5ffb3b..d5969a9 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # p_tqdm -[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/p_tqdm)](https://badge.fury.io/py/p_tqdm) -[![PyPI version](https://badge.fury.io/py/p_tqdm.svg)](https://badge.fury.io/py/p_tqdm) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/p-tqdm)](https://badge.fury.io/py/p-tqdm) +[![PyPI version](https://badge.fury.io/py/p-tqdm.svg)](https://badge.fury.io/py/p-tqdm) [![Build Status](https://travis-ci.org/swansonk14/p_tqdm.svg?branch=master)](https://travis-ci.org/swansonk14/p_tqdm) `p_tqdm` makes parallel processing with progress bars easy. From 0c8f179f8cc315a01e9c8327f1e10cde6062e6a6 Mon Sep 17 00:00:00 2001 From: Kyle Swanson Date: Sat, 14 Mar 2020 14:42:08 +0000 Subject: [PATCH 12/14] Fixing encoding issue in setup.py --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 69793ca..8f2d243 100644 --- a/setup.py +++ b/setup.py @@ -1,18 +1,18 @@ from setuptools import find_packages, setup -with open('README.md') as f: +with open('README.md', encoding='utf-8') as f: long_description = f.read() setup( name='p_tqdm', - version='1.3', + version='1.3.1', author='Kyle Swanson', author_email='swansonk.14@gmail.com', description='Parallel processing with progress bars', long_description=long_description, long_description_content_type='text/markdown', url='https://github.com/swansonk14/p_tqdm', - download_url='https://github.com/swansonk14/p_tqdm/v_1.3.tar.gz', + download_url='https://github.com/swansonk14/p_tqdm/v_1.3.1.tar.gz', license='MIT', packages=find_packages(), install_requires=[ From ce6d84ccd36968d10253d1ffed6d2c22b3c7b79c Mon Sep 17 00:00:00 2001 From: Kyle Swanson Date: Sat, 28 Mar 2020 23:20:29 -0400 Subject: [PATCH 13/14] Adding six dependency --- setup.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 8f2d243..e59c11f 100644 --- a/setup.py +++ b/setup.py @@ -5,19 +5,20 @@ setup( name='p_tqdm', - version='1.3.1', + version='1.3.2', author='Kyle Swanson', author_email='swansonk.14@gmail.com', description='Parallel processing with progress bars', long_description=long_description, long_description_content_type='text/markdown', url='https://github.com/swansonk14/p_tqdm', - download_url='https://github.com/swansonk14/p_tqdm/v_1.3.1.tar.gz', + download_url='https://github.com/swansonk14/p_tqdm/v_1.3.2.tar.gz', license='MIT', packages=find_packages(), install_requires=[ 'tqdm', - 'pathos' + 'pathos', + 'six' ], test_suite='nose.collector', tests_require=['nose'], From eb122f7622d22e3a3b03277a6d50b0d08339f5e7 Mon Sep 17 00:00:00 2001 From: Kyle Swanson Date: Thu, 9 Apr 2020 00:46:46 -0400 Subject: [PATCH 14/14] Fix to enable correct iteration over non-lists like numpy arrays; non-iterables now require partial --- LICENSE.txt | 2 +- README.md | 12 +++-- p_tqdm/p_tqdm.py | 131 +++++++++++++++-------------------------------- setup.py | 4 +- tests/tests.py | 61 +++++++--------------- 5 files changed, 71 insertions(+), 139 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index b2f35b8..eee0aea 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2019 Kyle Swanson +Copyright (c) 2020 Kyle Swanson Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index d5969a9..fb15e25 100644 --- a/README.md +++ b/README.md @@ -150,19 +150,25 @@ for result in iterator: ### Arguments -All `p_tqdm` functions accept any number of lists (of the same length) as input, as long as the number of lists matches the number of arguments of the function. Additionally, if any non-list variable is passed as an input to a `p_tqdm` function, the variable will be passed to all calls of the function. See the example below. +All `p_tqdm` functions accept any number of iterables as input, as long as the number of iterables matches the number of arguments of the function. + +To repeat a non-iterable argument along with the iterables, use Python's [partial](https://docs.python.org/3/library/functools.html#functools.partial) from the [functools](https://docs.python.org/3/library/functools.html) library. See the example below. ```python +from functools import partial + l1 = ['1', '2', '3'] l2 = ['a', 'b', 'c'] -def add(a, b, c): +def add(a, b, c=''): return a + b + c -added = p_map(add, l1, l2, '!') +added = p_map(partial(add, c='!'), l1, l2) # added == ['1a!', '2b!', '3c!'] ``` ### CPUs All the parallel `p_tqdm` functions can be passed the keyword `num_cpus` to indicate how many CPUs to use. The default is all CPUs. `num_cpus` can either be an integer to indicate the exact number of CPUs to use or a float to indicate the proportion of CPUs to use. + +Note that the parallel Pool objects used by `p_tqdm` are automatically closed when the map finishes processing. diff --git a/p_tqdm/p_tqdm.py b/p_tqdm/p_tqdm.py index 7109232..d4d3dc0 100644 --- a/p_tqdm/p_tqdm.py +++ b/p_tqdm/p_tqdm.py @@ -8,46 +8,29 @@ t_imap: Returns an iterator for a sequential map. """ -from typing import Any, Callable, Generator +from collections import Sized +from typing import Any, Callable, Generator, Iterable, List from pathos.helpers import cpu_count from pathos.multiprocessing import ProcessPool as Pool from tqdm.auto import tqdm -def _parallel(ordered: bool, function: Callable, *arrays: list, **kwargs: Any) -> Generator: +def _parallel(ordered: bool, function: Callable, *iterables: Iterable, **kwargs: Any) -> Generator: """Returns a generator for a parallel map with a progress bar. Arguments: ordered(bool): True for an ordered map, false for an unordered map. - function(Callable): The function to apply to each element - of the given arrays. - arrays(Tuple[list]): One or more arrays of the same length - containing the data to be mapped. If a non-list - variable is passed, it will be repeated a number - of times equal to the lengths of the list(s). If only - non-list variables are passed, the function will be - performed num_iter times. - num_cpus(int): The number of cpus to use in parallel. - If an int, uses that many cpus. - If a float, uses that proportion of cpus. - If None, uses all available cpus. - num_iter(int): If only non-list variables are passed, the - function will be performed num_iter times on - these variables. Default: 1. + function(Callable): The function to apply to each element of the given Iterables. + iterables(Tuple[Iterable]): One or more Iterables containing the data to be mapped. Returns: - A generator which will apply the function - to each element of the given arrays in - parallel in order with a progress bar. + A generator which will apply the function to each element of the given Iterables + in parallel in order with a progress bar. """ - # Convert tuple to list - arrays = list(arrays) - - # Extract kwargs + # Extract num_cpus num_cpus = kwargs.pop('num_cpus', None) - num_iter = kwargs.pop('num_iter', 1) # Determine num_cpus if num_cpus is None: @@ -55,124 +38,90 @@ def _parallel(ordered: bool, function: Callable, *arrays: list, **kwargs: Any) - elif type(num_cpus) == float: num_cpus = int(round(num_cpus * cpu_count())) - # Determine num_iter when at least one list is present - if any([type(array) == list for array in arrays]): - num_iter = max([len(array) for array in arrays if type(array) == list]) - - # Convert single variables to lists - # and confirm lists are same length - for i, array in enumerate(arrays): - if type(array) != list: - arrays[i] = [array for _ in range(num_iter)] - else: - assert len(array) == num_iter + # Determine length of tqdm (equal to length of shortest iterable) + length = min(len(iterable) for iterable in iterables if isinstance(iterable, Sized)) # Create parallel generator map_type = 'imap' if ordered else 'uimap' pool = Pool(num_cpus) map_func = getattr(pool, map_type) - for item in tqdm(map_func(function, *arrays), total=num_iter, **kwargs): + for item in tqdm(map_func(function, *iterables), total=length, **kwargs): yield item pool.clear() -def p_map(function: Callable, *arrays: list, **kwargs: Any) -> list: +def p_map(function: Callable, *iterables: Iterable, **kwargs: Any) -> List[Any]: """Performs a parallel ordered map with a progress bar.""" ordered = True - iterator = _parallel(ordered, function, *arrays, **kwargs) - result = list(iterator) + generator = _parallel(ordered, function, *iterables, **kwargs) + result = list(generator) return result -def p_imap(function: Callable, *arrays: list, **kwargs: Any) -> Generator: - """Returns an iterator for a parallel ordered map with a progress bar.""" +def p_imap(function: Callable, *iterables: Iterable, **kwargs: Any) -> Generator: + """Returns a generator for a parallel ordered map with a progress bar.""" ordered = True - iterator = _parallel(ordered, function, *arrays, **kwargs) + generator = _parallel(ordered, function, *iterables, **kwargs) - return iterator + return generator -def p_umap(function: Callable, *arrays: list, **kwargs: Any) -> list: +def p_umap(function: Callable, *iterables: Iterable, **kwargs: Any) -> List[Any]: """Performs a parallel unordered map with a progress bar.""" ordered = False - iterator = _parallel(ordered, function, *arrays, **kwargs) - result = list(iterator) + generator = _parallel(ordered, function, *iterables, **kwargs) + result = list(generator) return result -def p_uimap(function: Callable, *arrays: list, **kwargs: Any) -> Generator: - """Returns an iterator for a parallel unordered map with a progress bar.""" +def p_uimap(function: Callable, *iterables: Iterable, **kwargs: Any) -> Generator: + """Returns a generator for a parallel unordered map with a progress bar.""" ordered = False - iterator = _parallel(ordered, function, *arrays, **kwargs) + generator = _parallel(ordered, function, *iterables, **kwargs) - return iterator + return generator -def _sequential(function: Callable, *arrays: list, **kwargs: Any) -> Generator: +def _sequential(function: Callable, *iterables: Iterable, **kwargs: Any) -> Generator: """Returns a generator for a sequential map with a progress bar. Arguments: - function(Callable): The function to apply to each element - of the given arrays. - arrays(Tuple[list]): One or more arrays of the same length - containing the data to be mapped. If a non-list - variable is passed, it will be repeated a number - of times equal to the lengths of the list(s). If only - non-list variables are passed, the function will be - performed num_iter times. - num_iter(int): If only non-list variables are passed, the - function will be performed num_iter times on - these variables. Default: 1. + function(Callable): The function to apply to each element of the given Iterables. + iterables(Tuple[Iterable]): One or more Iterables containing the data to be mapped. Returns: - A generator which will apply the function - to each element of the given arrays sequentially - in order with a progress bar. + A generator which will apply the function to each element of the given Iterables + sequentially in order with a progress bar. """ - # Convert tuple to list - arrays = list(arrays) - - # Extract kwargs - num_iter = kwargs.pop('num_iter', 1) - - # Determine num_iter when at least one list is present - if any([type(array) == list for array in arrays]): - num_iter = max([len(array) for array in arrays if type(array) == list]) - - # Convert single variables to lists - # and confirm lists are same length - for i, array in enumerate(arrays): - if type(array) != list: - arrays[i] = [array for _ in range(num_iter)] - else: - assert len(array) == num_iter + # Determine length of tqdm (equal to length of shortest iterable) + length = min(len(iterable) for iterable in iterables if isinstance(iterable, Sized)) # Create sequential generator - for item in tqdm(map(function, *arrays), total=num_iter, **kwargs): + for item in tqdm(map(function, *iterables), total=length, **kwargs): yield item -def t_map(function: Callable, *arrays: list, **kwargs: Any) -> list: +def t_map(function: Callable, *iterables: Iterable, **kwargs: Any) -> List[Any]: """Performs a sequential map with a progress bar.""" - iterator = _sequential(function, *arrays, **kwargs) - result = list(iterator) + generator = _sequential(function, *iterables, **kwargs) + result = list(generator) return result -def t_imap(function: Callable, *arrays: list, **kwargs: Any) -> Generator: - """Returns an iterator for a sequential map with a progress bar.""" +def t_imap(function: Callable, *iterables: Iterable, **kwargs: Any) -> Generator: + """Returns a generator for a sequential map with a progress bar.""" - iterator = _sequential(function, *arrays, **kwargs) + generator = _sequential(function, *iterables, **kwargs) - return iterator + return generator diff --git a/setup.py b/setup.py index e59c11f..8a9d93f 100644 --- a/setup.py +++ b/setup.py @@ -5,14 +5,14 @@ setup( name='p_tqdm', - version='1.3.2', + version='1.3.3', author='Kyle Swanson', author_email='swansonk.14@gmail.com', description='Parallel processing with progress bars', long_description=long_description, long_description_content_type='text/markdown', url='https://github.com/swansonk14/p_tqdm', - download_url='https://github.com/swansonk14/p_tqdm/v_1.3.2.tar.gz', + download_url='https://github.com/swansonk14/p_tqdm/v_1.3.3.tar.gz', license='MIT', packages=find_packages(), install_requires=[ diff --git a/tests/tests.py b/tests/tests.py index 7f063cc..956494d 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,4 +1,5 @@ import unittest +from functools import partial from p_tqdm import p_map, p_imap, p_umap, p_uimap, t_map, t_imap @@ -11,8 +12,8 @@ def add_2(a, b): return a + b -def add_3(a, b, c): - return a + b + c +def add_3(a, b, c=0): + return a + 2 * b + 3 * c class Test_p_map(unittest.TestCase): @@ -51,11 +52,11 @@ def test_two_lists_and_one_single(self): array_1 = [1, 2, 3] array_2 = [10, 11, 12] single = 5 - result = self.func(add_3, array_1, single, array_2) + result = self.func(partial(add_3, single), array_1, array_2) if self.generator: result = list(result) - correct_array = [16, 18, 20] + correct_array = [37, 42, 47] if self.ordered: self.assertEqual(correct_array, result) else: @@ -65,63 +66,39 @@ def test_one_list_and_two_singles(self): array = [1, 2, 3] single_1 = 5 single_2 = -2 - result = self.func(add_3, single_1, array, single_2) + result = self.func(partial(add_3, single_1, c=single_2), array) if self.generator: result = list(result) - correct_array = [4, 5, 6] + correct_array = [1, 3, 5] if self.ordered: self.assertEqual(correct_array, result) else: self.assertEqual(sorted(correct_array), sorted(result)) - def test_one_single(self): - single = 5 - result = self.func(add_1, single) - if self.generator: - result = list(result) - - correct_array = [6] - if self.ordered: - self.assertEqual(correct_array, result) - else: - self.assertEqual(sorted(correct_array), sorted(result)) - - def test_one_single_with_num_iter(self): - single = 5 - num_iter = 3 - result = self.func(add_1, single, num_iter=num_iter) - if self.generator: - result = list(result) - - correct_array = [6] * num_iter - if self.ordered: - self.assertEqual(correct_array, result) - else: - self.assertEqual(sorted(correct_array), sorted(result)) - - def test_two_singles(self): - single_1 = 5 - single_2 = -2 - result = self.func(add_2, single_1, single_2) + def test_list_and_generator_and_single_equal_length(self): + array = [1, 2, 3] + generator = range(3) + single = -3 + result = self.func(partial(add_3, c=single), array, generator) if self.generator: result = list(result) - correct_array = [3] + correct_array = [-8, -5, -2] if self.ordered: self.assertEqual(correct_array, result) else: self.assertEqual(sorted(correct_array), sorted(result)) - def test_two_singles_with_num_iter(self): - single_1 = 5 - single_2 = -2 - num_iter = 3 - result = self.func(add_2, single_1, single_2, num_iter=num_iter) + def test_list_and_generator_and_single_unequal_length(self): + array = [1, 2, 3, 4, 5, 6] + generator = range(3) + single = -3 + result = self.func(partial(add_3, c=single), array, generator) if self.generator: result = list(result) - correct_array = [3] * num_iter + correct_array = [-8, -5, -2] if self.ordered: self.assertEqual(correct_array, result) else: