diff --git a/README.md b/README.md index e585d78..807d35e 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ -QR +QR3 ===== -**QR** helps you create and work with **queue, capped collection (bounded queue), deque, and stack** data structures for **Redis**. +**QR3** helps you create and work with **queue, capped collection (bounded queue), deque, and stack** data structures for **Redis**. Redis is well-suited for implementations of these abstract data structures, and QR makes it even easier to work with the structures in Python. Quick Setup @@ -25,6 +25,23 @@ Then install `qr`: python setup.py install ``` +To run tests: +``` +python -m unittest discover -v +``` + +Responding to PR's +------------------ +Given that this package primarily supports internal use cases, we cannot guarantee a +specific response time on PRs for new features. However, we will do our best to +consider them in a timely fashion. + +We do commit to reviewing anything related to a security issue in a timely manner. +We ask that you first submit anything of that nature to security@doctorondemand.com +prior to creating a PR and follow responsible disclosure rules. + +Thanks for your interest in helping with this package! + Basics of QR ------------------ diff --git a/qr3/__init__.py b/qr3/__init__.py new file mode 100644 index 0000000..6d61f01 --- /dev/null +++ b/qr3/__init__.py @@ -0,0 +1 @@ +"""This package contains the qr classes.""" diff --git a/qr.py b/qr3/qr.py similarity index 94% rename from qr.py rename to qr3/qr.py index 486c043..5c52d52 100644 --- a/qr.py +++ b/qr3/qr.py @@ -1,13 +1,19 @@ """ + QR | Redis-Based Data Structures in Python """ +from future import standard_library +standard_library.install_aliases() +from builtins import object __author__ = 'Ted Nyman' -__version__ = '0.6.0' +__version__ = '1.0.0' __license__ = 'MIT' -import redis import logging +import pickle +import redis + try: import json @@ -17,10 +23,7 @@ # This is a complete nod to hotqueue -- this is one of the # things that they did right. Natively pickling and unpiclking # objects is pretty useful. -try: - import cPickle as pickle -except ImportError: - import pickle + class NullHandler(logging.Handler): """A logging handler that discards all logging records""" @@ -36,6 +39,7 @@ def emit(self, record): # connections connectionPools = {} + def getRedis(**kwargs): """ Match up the provided kwargs with an existing connection pool. @@ -44,7 +48,7 @@ def getRedis(**kwargs): connection pool mechanism to keep the number of open file descriptors tractable. """ - key = ':'.join((repr(key) + '=>' + repr(value)) for key, value in kwargs.items()) + key = ':'.join((repr(key) + '=>' + repr(value)) for key, value in list(kwargs.items())) try: return redis.Redis(connection_pool=connectionPools[key]) except KeyError: @@ -52,6 +56,7 @@ def getRedis(**kwargs): connectionPools[key] = cp return redis.Redis(connection_pool=cp) + class worker(object): def __init__(self, q, err=None, *args, **kwargs): self.q = q @@ -78,7 +83,8 @@ def wrapped(): except: pass return wrapped - + + class BaseQueue(object): """Base functionality common to queues""" @staticmethod @@ -90,7 +96,7 @@ def __init__(self, key, **kwargs): self.serializer = pickle self.redis = getRedis(**kwargs) self.key = key - + def __len__(self): """Return the length of the queue""" return self.redis.llen(self.key) @@ -134,15 +140,15 @@ def load(self, fobj): def dumpfname(self, fname, truncate=False): """Destructively dump the contents of the queue into fname""" if truncate: - with file(fname, 'w+') as f: + with open(fname, 'w+') as f: self.dump(f) else: - with file(fname, 'a+') as f: + with open(fname, 'a+') as f: self.dump(f) def loadfname(self, fname): """Load the contents of the contents of fname into the queue""" - with file(fname) as f: + with open(fname) as f: self.load(f) def extend(self, vals): @@ -198,6 +204,7 @@ def pop_back(self): log.debug('Popped ** %s ** from key ** %s **' % (popped, self.key)) return self._unpack(popped) + class Queue(BaseQueue): """Implements a FIFO queue""" @@ -218,7 +225,8 @@ def pop(self, block=False): queue, popped = self.redis.brpop(self.key) log.debug('Popped ** %s ** from key ** %s **' % (popped, self.key)) return self._unpack(popped) - + + class PriorityQueue(BaseQueue): """A priority queue""" def __len__(self): @@ -240,39 +248,39 @@ def __getitem__(self, val): def dump(self, fobj): """Destructively dump the contents of the queue into fp""" - next = self.pop() - while next: - self.serializer.dump(next[0], fobj) - next = self.pop() - + next = self.pop(True) + while next[0] is not None: + self.serializer.dump(next, fobj) + next = self.pop(True) + def load(self, fobj): """Load the contents of the provided fobj into the queue""" try: while True: value, score = self.serializer.load(fobj) - self.redis.zadd(self.key, value, score) + self.push(value, score) except Exception as e: return def dumpfname(self, fname, truncate=False): """Destructively dump the contents of the queue into fname""" if truncate: - with file(fname, 'w+') as f: + with open(fname, 'w+') as f: self.dump(f) else: - with file(fname, 'a+') as f: + with open(fname, 'a+') as f: self.dump(f) def loadfname(self, fname): """Load the contents of the contents of fname into the queue""" - with file(fname) as f: + with open(fname) as f: self.load(f) def extend(self, vals): """Extends the elements in the queue.""" with self.redis.pipeline(transaction=False) as pipe: for val, score in vals: - pipe.zadd(self.key, self._pack(val), score) + pipe.zadd(self.key, {self._pack(val): score}) return pipe.execute() def peek(self, withscores=False): @@ -310,7 +318,8 @@ def pop(self, withscores=False): def push(self, value, score): '''Add an element with a given score''' - return self.redis.zadd(self.key, self._pack(value), score) + return self.redis.zadd(self.key, {self._pack(value): score}) + class CappedCollection(BaseQueue): """ @@ -349,6 +358,7 @@ def pop(self, block=False): log.debug('Popped ** %s ** from key ** %s **' % (popped, self.key)) return self._unpack(popped) + class Stack(BaseQueue): """Implements a LIFO stack""" diff --git a/setup.py b/setup.py index 5e2152a..680cb7f 100644 --- a/setup.py +++ b/setup.py @@ -1,21 +1,19 @@ #!/usr/bin/env python -import os -import unittest from setuptools import setup, find_packages -version = '0.6.0' +version = '1.0.1' LONG_DESCRIPTION = ''' -Full documentation (with example code) is at http://github.com/tnm/qr +Full documentation (with example code) is at http://github.com/doctorondemand/qr3 -QR +QR3 ===== -**QR** helps you create and work with **queue, capped collection (bounded queue), -deque, and stack** data structures for **Redis**. Redis is well-suited for -implementations of these abstract data structures, and QR makes it even easier to +**QR3** helps you create and work with **queue, capped collection (bounded queue), +deque, and stack** data structures for **Redis**. Redis is well-suited for +implementations of these abstract data structures, and QR3 makes it even easier to work with the structures in Python. Quick Setup @@ -24,26 +22,41 @@ of MULTI/EXEC, so you'll need the Git edge version), and the current Python interface for Redis, [redis-py](http://github.com/andymccurdy/redis-py "redis-py"). -Run setup.py to install qr. +Run setup.py to install qr3 or 'pip install qr3'. +Responding to PR's +------------------ +Given that this package primarily supports internal use cases, we cannot guarantee a +specific response time on PRs for new features. However, we will do our best to +consider them in a timely fashion. + +We do commit to reviewing anything related to a security issue in a timely manner. +We ask that you first submit anything of that nature to security@doctorondemand.com +prior to creating a PR and follow responsible disclosure rules. + +Thanks for your interest in helping with this package! ''' setup( - name = 'qr', + name = 'qr3', version = version, description = 'Redis-powered queues, capped collections, deques, and stacks', long_description = LONG_DESCRIPTION, - url = 'http://github.com/tnm/qr', + url = 'http://github.com/doctorondemand/qr3', author = 'Ted Nyman', author_email = 'ted@ted.io', + maintainer = 'DoctorOnDemand', + maintainer_email = 'sustaining@doctorondemand.com', keywords = 'Redis, queue, data structures', license = 'MIT', packages = find_packages(), + install_requires = ['future', 'redis>=3.0.0'], py_modules = ['qr'], include_package_data = True, zip_safe = False, classifiers = [ - 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', diff --git a/test/tests.py b/test/test_qr3.py similarity index 63% rename from test/tests.py rename to test/test_qr3.py index 6520397..6a296b5 100644 --- a/test/tests.py +++ b/test/test_qr3.py @@ -1,44 +1,47 @@ -import os -import qr +from builtins import zip +from builtins import range +from qr3 import qr import redis +import tempfile import unittest r = redis.Redis() + class Queue(unittest.TestCase): def setUp(self): r.delete('qrtestqueue') self.q = qr.Queue(key='qrtestqueue') - self.assertEquals(len(self.q), 0) + self.assertEqual(len(self.q), 0) def test_roundtrip(self): q = self.q q.push('foo') - self.assertEquals(len(q), 1) - self.assertEquals(q.pop(), 'foo') - self.assertEquals(len(q), 0) + self.assertEqual(len(q), 1) + self.assertEqual(q.pop(), 'foo') + self.assertEqual(len(q), 0) def test_order(self): q = self.q q.push('foo') q.push('bar') - self.assertEquals(q.pop(), 'foo') - self.assertEquals(q.pop(), 'bar') + self.assertEqual(q.pop(), 'foo') + self.assertEqual(q.pop(), 'bar') def test_order_mixed(self): q = self.q q.push('foo') - self.assertEquals(q.pop(), 'foo') + self.assertEqual(q.pop(), 'foo') q.push('bar') - self.assertEquals(q.pop(), 'bar') + self.assertEqual(q.pop(), 'bar') def test_len(self): count = 100 for i in range(count): - self.assertEquals(len(self.q), i) + self.assertEqual(len(self.q), i) self.q.push(i) for i in range(count): - self.assertEquals(len(self.q), count - i) + self.assertEqual(len(self.q), count - i) self.q.pop() self.q.clear() @@ -50,29 +53,29 @@ def test_get_item(self): items.reverse() # Get single values for i in range(count): - self.assertEquals(self.q[i], items[i]) + self.assertEqual(self.q[i], items[i]) # Get small ranges for i in range(count-1): - self.assertEquals(self.q[i:i+1], items[i:i+1]) + self.assertEqual(self.q[i:i+1], items[i:i+1]) # Now get the whole range - self.assertEquals(self.q[0:-1], items[0:-1]) + self.assertEqual(self.q[0:-1], items[0:-1]) self.q.clear() - + def test_extend(self): '''Test extending a queue, including with a generator''' count = 100 self.q.extend(i for i in range(count)) - self.assertEquals(len(self.q), count) + self.assertEqual(len(self.q), count) self.q.clear() - + self.q.extend([i for i in range(count)]) - self.assertEquals(len(self.q), count) + self.assertEqual(len(self.q), count) self.q.clear() - - self.q.extend(range(count)) - self.assertEquals(self.q.elements(), [count - i - 1 for i in range(count)]) + + self.q.extend(list(range(count))) + self.assertEqual(self.q.elements(), [count - i - 1 for i in range(count)]) self.q.clear() - + def test_pack_unpack(self): '''Make sure that it behaves like python-object-in, python-object-out''' count = 100 @@ -81,71 +84,71 @@ def test_pack_unpack(self): while next: self.assertTrue(isinstance(next, dict)) next = self.q.pop() - + def test_dump_load(self): # Get a temporary file to dump a queue to that file count = 100 - self.q.extend(range(count)) - self.assertEquals(self.q.elements(), [count - i - 1for i in range(count)]) - with os.tmpfile() as f: + self.q.extend(list(range(count))) + self.assertEqual(self.q.elements(), [count - i - 1 for i in range(count)]) + with tempfile.TemporaryFile() as f: self.q.dump(f) # Now, assert that it is empty - self.assertEquals(len(self.q), 0) + self.assertEqual(len(self.q), 0) # Now, try to load it back in f.seek(0) self.q.load(f) - self.assertEquals(len(self.q), count) - self.assertEquals(self.q.elements(), [count - i - 1 for i in range(count)]) + self.assertEqual(len(self.q), count) + self.assertEqual(self.q.elements(), [count - i - 1 for i in range(count)]) # Now clean up after myself f.truncate() self.q.clear() - + class CappedCollection(unittest.TestCase): def setUp(self): r.delete('qrtestcc') self.aq = qr.CappedCollection(key='qrtestcc', size=3) - self.assertEquals(len(self.aq), 0) + self.assertEqual(len(self.aq), 0) def test_roundtrip(self): aq = self.aq aq.push('foo') - self.assertEquals(len(aq), 1) - self.assertEquals(aq.pop(), 'foo') - self.assertEquals(len(aq), 0) + self.assertEqual(len(aq), 1) + self.assertEqual(aq.pop(), 'foo') + self.assertEqual(len(aq), 0) def test_order(self): aq = self.aq aq.push('foo') aq.push('bar') - self.assertEquals(aq.pop(), 'foo') - self.assertEquals(aq.pop(), 'bar') + self.assertEqual(aq.pop(), 'foo') + self.assertEqual(aq.pop(), 'bar') def test_order_mixed(self): aq = self.aq aq.push('foo') - self.assertEquals(aq.pop(), 'foo') + self.assertEqual(aq.pop(), 'foo') aq.push('bar') - self.assertEquals(aq.pop(), 'bar') + self.assertEqual(aq.pop(), 'bar') def test_limit(self): aq = self.aq aq.push('a') aq.push('b') aq.push('c') - self.assertEquals(len(aq), 3) + self.assertEqual(len(aq), 3) aq.push('d') aq.push('e') - self.assertEquals(len(aq), 3) - self.assertEquals(aq.pop(), 'c') - self.assertEquals(aq.pop(), 'd') - self.assertEquals(aq.pop(), 'e') - self.assertEquals(len(aq), 0) + self.assertEqual(len(aq), 3) + self.assertEqual(aq.pop(), 'c') + self.assertEqual(aq.pop(), 'd') + self.assertEqual(aq.pop(), 'e') + self.assertEqual(len(aq), 0) def test_extend(self): '''Test extending a queue, including with a generator''' count = 100 self.aq.extend(i for i in range(count)) - self.assertEquals(len(self.aq), self.aq.size) + self.assertEqual(len(self.aq), self.aq.size) self.aq.clear() class Stack(unittest.TestCase): @@ -156,23 +159,23 @@ def setUp(self): def test_roundtrip(self): stack = self.stack stack.push('foo') - self.assertEquals(len(stack), 1) - self.assertEquals(stack.pop(), 'foo') - self.assertEquals(len(stack), 0) + self.assertEqual(len(stack), 1) + self.assertEqual(stack.pop(), 'foo') + self.assertEqual(len(stack), 0) def test_order(self): stack = self.stack stack.push('foo') stack.push('bar') - self.assertEquals(stack.pop(), 'bar') - self.assertEquals(stack.pop(), 'foo') + self.assertEqual(stack.pop(), 'bar') + self.assertEqual(stack.pop(), 'foo') def test_order_mixed(self): stack = self.stack stack.push('foo') - self.assertEquals(stack.pop(), 'foo') + self.assertEqual(stack.pop(), 'foo') stack.push('bar') - self.assertEquals(stack.pop(), 'bar') + self.assertEqual(stack.pop(), 'bar') def test_get_item(self): count = 100 @@ -181,42 +184,44 @@ def test_get_item(self): items.reverse() # Get single values for i in range(count): - self.assertEquals(self.stack[i], items[i]) + self.assertEqual(self.stack[i], items[i]) # Get small ranges for i in range(count-1): - self.assertEquals(self.stack[i:i+2], items[i:i+2]) + self.assertEqual(self.stack[i:i+2], items[i:i+2]) # Now get the whole range - self.assertEquals(self.stack[0:-1], items[0:-1]) + self.assertEqual(self.stack[0:-1], items[0:-1]) self.stack.clear() def test_extend(self): '''Test extending a queue, including with a generator''' count = 100 self.stack.extend(i for i in range(count)) - self.assertEquals(self.stack.elements(), [count - i - 1 for i in range(count)]) - + self.assertEqual(self.stack.elements(), [count - i - 1 for i in range(count)]) + # Also, make sure it's still a stack. It should be in reverse order last = self.stack.pop() - while last != None: + now = None + while last is not None: now = self.stack.pop() - self.assertTrue(last > now) + if now is not None: + self.assertTrue(last > now) last = now self.stack.clear() def test_dump_load(self): # Get a temporary file to dump a queue to that file count = 100 - self.stack.extend(range(count)) - self.assertEquals(self.stack.elements(), [count - i - 1 for i in range(count)]) - with os.tmpfile() as f: + self.stack.extend(list(range(count))) + self.assertEqual(self.stack.elements(), [count - i - 1 for i in range(count)]) + with tempfile.TemporaryFile() as f: self.stack.dump(f) # Now, assert that it is empty - self.assertEquals(len(self.stack), 0) + self.assertEqual(len(self.stack), 0) # Now, try to load it back in f.seek(0) self.stack.load(f) - self.assertEquals(len(self.stack), count) - self.assertEquals(self.stack.elements(), [count - i - 1 for i in range(count)]) + self.assertEqual(len(self.stack), count) + self.assertEqual(self.stack.elements(), [count - i - 1 for i in range(count)]) # Now clean up after myself f.truncate() self.stack.clear() @@ -228,56 +233,56 @@ def setUp(self): def test_roundtrip(self): self.q.push('foo', 1) - self.assertEquals(len(self.q), 1) - self.assertEquals(self.q.pop(), 'foo') - self.assertEquals(len(self.q), 0) + self.assertEqual(len(self.q), 1) + self.assertEqual(self.q.pop(), 'foo') + self.assertEqual(len(self.q), 0) def test_order(self): self.q.push('foo', 1) self.q.push('bar', 0) - self.assertEquals(self.q.pop(), 'bar') - self.assertEquals(self.q.pop(), 'foo') + self.assertEqual(self.q.pop(), 'bar') + self.assertEqual(self.q.pop(), 'foo') def test_get_item(self): count = 100 items = [i for i in range(count)] - self.q.extend(zip(items, items)) + self.q.extend(list(zip(items, items))) # Get single values for i in range(count): - self.assertEquals(self.q[i], items[i]) + self.assertEqual(self.q[i], items[i]) # Get small ranges for i in range(count-1): - self.assertEquals(self.q[i:i+2], items[i:i+2]) + self.assertEqual(self.q[i:i+2], items[i:i+2]) # Now get the whole range - self.assertEquals(self.q[0:-1], items[0:-1]) + self.assertEqual(self.q[0:-1], items[0:-1]) self.q.clear() def test_extend(self): '''Test extending a queue, including with a generator''' count = 100 items = [i for i in range(count)] - self.q.extend(zip(items, items)) - self.assertEquals(self.q.elements(), items) + self.q.extend(list(zip(items, items))) + self.assertEqual(self.q.elements(), items) self.q.clear() def test_pop(self): '''Test whether or not we can get real values with pop''' count = 100 items = [i for i in range(count)] - self.q.extend(zip(items, items)) + self.q.extend(list(zip(items, items))) next = self.q.pop() while next: self.assertTrue(isinstance(next, int)) next = self.q.pop() # Now we'll pop with getting the scores as well items = [i for i in range(count)] - self.q.extend(zip(items, items)) + self.q.extend(list(zip(items, items))) value, score = self.q.pop(withscores=True) while value: self.assertTrue(isinstance(value, int)) self.assertTrue(isinstance(score, float)) value, score = self.q.pop(withscores=True) - + def test_push(self): '''Test whether we can push well''' count = 100 @@ -287,33 +292,34 @@ def test_push(self): while value: self.assertEqual(value + score, count) value, score = self.q.pop(withscores=True) - + def test_uniqueness(self): count = 100 # Push the same value on with different scores for i in range(count): self.q.push(1, i) - self.assertEquals(len(self.q), 1) + self.assertEqual(len(self.q), 1) self.q.clear() - + def test_dump_load(self): # Get a temporary file to dump a queue to that file count = 100 items = [i for i in range(count)] - self.q.extend(zip(items, items)) - self.assertEquals(self.q.elements(), items) - with os.tmpfile() as f: + self.q.extend(list(zip(items, items))) + self.assertEqual(self.q.elements(), items) + with tempfile.TemporaryFile() as f: self.q.dump(f) # Now, assert that it is empty - self.assertEquals(len(self.q), 0) + self.assertEqual(len(self.q), 0) # Now, try to load it back in f.seek(0) self.q.load(f) - self.assertEquals(len(self.q), count) - self.assertEquals(self.q.elements(), items) + self.assertEqual(len(self.q), count) + self.assertEqual(self.q.elements(), items) # Now clean up after myself f.truncate() self.q.clear() + if __name__ == '__main__': unittest.main()