From 8457e91329f2dedb1e105a494eb9cb3d4b649b05 Mon Sep 17 00:00:00 2001 From: Vinay Valsaraj Date: Fri, 6 Jul 2018 11:01:10 -0700 Subject: [PATCH 01/13] Futurize ran on both qr.py and tests.py --- qr.py | 7 +++++-- test/tests.py | 18 ++++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/qr.py b/qr.py index 486c043..de4dd28 100644 --- a/qr.py +++ b/qr.py @@ -2,6 +2,9 @@ QR | Redis-Based Data Structures in Python """ +from future import standard_library +standard_library.install_aliases() +from builtins import object __author__ = 'Ted Nyman' __version__ = '0.6.0' __license__ = 'MIT' @@ -18,7 +21,7 @@ # things that they did right. Natively pickling and unpiclking # objects is pretty useful. try: - import cPickle as pickle + import pickle as pickle except ImportError: import pickle @@ -44,7 +47,7 @@ def getRedis(**kwargs): connection pool mechanism to keep the number of open file descriptors tractable. """ - key = ':'.join((repr(key) + '=>' + repr(value)) for key, value in kwargs.items()) + key = ':'.join((repr(key) + '=>' + repr(value)) for key, value in list(kwargs.items())) try: return redis.Redis(connection_pool=connectionPools[key]) except KeyError: diff --git a/test/tests.py b/test/tests.py index 6520397..315968d 100644 --- a/test/tests.py +++ b/test/tests.py @@ -1,3 +1,5 @@ +from builtins import zip +from builtins import range import os import qr import redis @@ -69,7 +71,7 @@ def test_extend(self): self.assertEquals(len(self.q), count) self.q.clear() - self.q.extend(range(count)) + self.q.extend(list(range(count))) self.assertEquals(self.q.elements(), [count - i - 1 for i in range(count)]) self.q.clear() @@ -85,7 +87,7 @@ def test_pack_unpack(self): def test_dump_load(self): # Get a temporary file to dump a queue to that file count = 100 - self.q.extend(range(count)) + self.q.extend(list(range(count))) self.assertEquals(self.q.elements(), [count - i - 1for i in range(count)]) with os.tmpfile() as f: self.q.dump(f) @@ -206,7 +208,7 @@ def test_extend(self): def test_dump_load(self): # Get a temporary file to dump a queue to that file count = 100 - self.stack.extend(range(count)) + self.stack.extend(list(range(count))) self.assertEquals(self.stack.elements(), [count - i - 1 for i in range(count)]) with os.tmpfile() as f: self.stack.dump(f) @@ -241,7 +243,7 @@ def test_order(self): def test_get_item(self): count = 100 items = [i for i in range(count)] - self.q.extend(zip(items, items)) + self.q.extend(list(zip(items, items))) # Get single values for i in range(count): self.assertEquals(self.q[i], items[i]) @@ -256,7 +258,7 @@ def test_extend(self): '''Test extending a queue, including with a generator''' count = 100 items = [i for i in range(count)] - self.q.extend(zip(items, items)) + self.q.extend(list(zip(items, items))) self.assertEquals(self.q.elements(), items) self.q.clear() @@ -264,14 +266,14 @@ def test_pop(self): '''Test whether or not we can get real values with pop''' count = 100 items = [i for i in range(count)] - self.q.extend(zip(items, items)) + self.q.extend(list(zip(items, items))) next = self.q.pop() while next: self.assertTrue(isinstance(next, int)) next = self.q.pop() # Now we'll pop with getting the scores as well items = [i for i in range(count)] - self.q.extend(zip(items, items)) + self.q.extend(list(zip(items, items))) value, score = self.q.pop(withscores=True) while value: self.assertTrue(isinstance(value, int)) @@ -300,7 +302,7 @@ def test_dump_load(self): # Get a temporary file to dump a queue to that file count = 100 items = [i for i in range(count)] - self.q.extend(zip(items, items)) + self.q.extend(list(zip(items, items))) self.assertEquals(self.q.elements(), items) with os.tmpfile() as f: self.q.dump(f) From e03775c2ed12be7efb9d851e2191f42058d45b54 Mon Sep 17 00:00:00 2001 From: Vinay Valsaraj Date: Fri, 6 Jul 2018 15:00:04 -0700 Subject: [PATCH 02/13] Fixed bug in dump where 0 was ending the loop and load where loaded contents were not being packed before put in redis. --- qr.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/qr.py b/qr.py index de4dd28..cec7630 100644 --- a/qr.py +++ b/qr.py @@ -243,17 +243,17 @@ def __getitem__(self, val): def dump(self, fobj): """Destructively dump the contents of the queue into fp""" - next = self.pop() - while next: - self.serializer.dump(next[0], fobj) - next = self.pop() - + next = self.pop(True) + while next[0] is not None: + self.serializer.dump(next, fobj) + next = self.pop(True) + def load(self, fobj): """Load the contents of the provided fobj into the queue""" try: while True: value, score = self.serializer.load(fobj) - self.redis.zadd(self.key, value, score) + self.push(value, score) except Exception as e: return From 4f9005d2286f8bacc7288d57c54ed463f2770f0e Mon Sep 17 00:00:00 2001 From: Vinay Valsaraj Date: Fri, 6 Jul 2018 15:00:39 -0700 Subject: [PATCH 03/13] Fix syntax error in test. --- test/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tests.py b/test/tests.py index 315968d..feef1e7 100644 --- a/test/tests.py +++ b/test/tests.py @@ -88,7 +88,7 @@ def test_dump_load(self): # Get a temporary file to dump a queue to that file count = 100 self.q.extend(list(range(count))) - self.assertEquals(self.q.elements(), [count - i - 1for i in range(count)]) + self.assertEquals(self.q.elements(), [count - i - 1 for i in range(count)]) with os.tmpfile() as f: self.q.dump(f) # Now, assert that it is empty From 45b31f8aa551f3e69b1073e12cc12884b5882917 Mon Sep 17 00:00:00 2001 From: Vinay Valsaraj Date: Fri, 6 Jul 2018 15:11:12 -0700 Subject: [PATCH 04/13] Refactored tests for py3 compatibility. --- test/tests.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/tests.py b/test/tests.py index feef1e7..d9cc39b 100644 --- a/test/tests.py +++ b/test/tests.py @@ -1,8 +1,8 @@ from builtins import zip from builtins import range -import os import qr import redis +import tempfile import unittest r = redis.Redis() @@ -89,7 +89,7 @@ def test_dump_load(self): count = 100 self.q.extend(list(range(count))) self.assertEquals(self.q.elements(), [count - i - 1 for i in range(count)]) - with os.tmpfile() as f: + with tempfile.TemporaryFile() as f: self.q.dump(f) # Now, assert that it is empty self.assertEquals(len(self.q), 0) @@ -199,9 +199,11 @@ def test_extend(self): # Also, make sure it's still a stack. It should be in reverse order last = self.stack.pop() - while last != None: + now = None + while last is not None: now = self.stack.pop() - self.assertTrue(last > now) + if now is not None: + self.assertTrue(last > now) last = now self.stack.clear() @@ -210,7 +212,7 @@ def test_dump_load(self): count = 100 self.stack.extend(list(range(count))) self.assertEquals(self.stack.elements(), [count - i - 1 for i in range(count)]) - with os.tmpfile() as f: + with tempfile.TemporaryFile() as f: self.stack.dump(f) # Now, assert that it is empty self.assertEquals(len(self.stack), 0) @@ -304,7 +306,7 @@ def test_dump_load(self): items = [i for i in range(count)] self.q.extend(list(zip(items, items))) self.assertEquals(self.q.elements(), items) - with os.tmpfile() as f: + with tempfile.TemporaryFile() as f: self.q.dump(f) # Now, assert that it is empty self.assertEquals(len(self.q), 0) From c23631b4bb19d563f11ff44a24b3212c879061d4 Mon Sep 17 00:00:00 2001 From: Vinay Valsaraj Date: Fri, 6 Jul 2018 15:23:58 -0700 Subject: [PATCH 05/13] AssertEquals changed to AssertEqual and file replaced with open for py3 compatibility. --- qr.py | 12 ++--- test/tests.py | 128 +++++++++++++++++++++++++------------------------- 2 files changed, 70 insertions(+), 70 deletions(-) diff --git a/qr.py b/qr.py index cec7630..294aae9 100644 --- a/qr.py +++ b/qr.py @@ -137,15 +137,15 @@ def load(self, fobj): def dumpfname(self, fname, truncate=False): """Destructively dump the contents of the queue into fname""" if truncate: - with file(fname, 'w+') as f: + with open(fname, 'w+') as f: self.dump(f) else: - with file(fname, 'a+') as f: + with open(fname, 'a+') as f: self.dump(f) def loadfname(self, fname): """Load the contents of the contents of fname into the queue""" - with file(fname) as f: + with open(fname) as f: self.load(f) def extend(self, vals): @@ -260,15 +260,15 @@ def load(self, fobj): def dumpfname(self, fname, truncate=False): """Destructively dump the contents of the queue into fname""" if truncate: - with file(fname, 'w+') as f: + with open(fname, 'w+') as f: self.dump(f) else: - with file(fname, 'a+') as f: + with open(fname, 'a+') as f: self.dump(f) def loadfname(self, fname): """Load the contents of the contents of fname into the queue""" - with file(fname) as f: + with open(fname) as f: self.load(f) def extend(self, vals): diff --git a/test/tests.py b/test/tests.py index d9cc39b..331121f 100644 --- a/test/tests.py +++ b/test/tests.py @@ -11,36 +11,36 @@ class Queue(unittest.TestCase): def setUp(self): r.delete('qrtestqueue') self.q = qr.Queue(key='qrtestqueue') - self.assertEquals(len(self.q), 0) + self.assertEqual(len(self.q), 0) def test_roundtrip(self): q = self.q q.push('foo') - self.assertEquals(len(q), 1) - self.assertEquals(q.pop(), 'foo') - self.assertEquals(len(q), 0) + self.assertEqual(len(q), 1) + self.assertEqual(q.pop(), 'foo') + self.assertEqual(len(q), 0) def test_order(self): q = self.q q.push('foo') q.push('bar') - self.assertEquals(q.pop(), 'foo') - self.assertEquals(q.pop(), 'bar') + self.assertEqual(q.pop(), 'foo') + self.assertEqual(q.pop(), 'bar') def test_order_mixed(self): q = self.q q.push('foo') - self.assertEquals(q.pop(), 'foo') + self.assertEqual(q.pop(), 'foo') q.push('bar') - self.assertEquals(q.pop(), 'bar') + self.assertEqual(q.pop(), 'bar') def test_len(self): count = 100 for i in range(count): - self.assertEquals(len(self.q), i) + self.assertEqual(len(self.q), i) self.q.push(i) for i in range(count): - self.assertEquals(len(self.q), count - i) + self.assertEqual(len(self.q), count - i) self.q.pop() self.q.clear() @@ -52,27 +52,27 @@ def test_get_item(self): items.reverse() # Get single values for i in range(count): - self.assertEquals(self.q[i], items[i]) + self.assertEqual(self.q[i], items[i]) # Get small ranges for i in range(count-1): - self.assertEquals(self.q[i:i+1], items[i:i+1]) + self.assertEqual(self.q[i:i+1], items[i:i+1]) # Now get the whole range - self.assertEquals(self.q[0:-1], items[0:-1]) + self.assertEqual(self.q[0:-1], items[0:-1]) self.q.clear() def test_extend(self): '''Test extending a queue, including with a generator''' count = 100 self.q.extend(i for i in range(count)) - self.assertEquals(len(self.q), count) + self.assertEqual(len(self.q), count) self.q.clear() self.q.extend([i for i in range(count)]) - self.assertEquals(len(self.q), count) + self.assertEqual(len(self.q), count) self.q.clear() self.q.extend(list(range(count))) - self.assertEquals(self.q.elements(), [count - i - 1 for i in range(count)]) + self.assertEqual(self.q.elements(), [count - i - 1 for i in range(count)]) self.q.clear() def test_pack_unpack(self): @@ -88,16 +88,16 @@ def test_dump_load(self): # Get a temporary file to dump a queue to that file count = 100 self.q.extend(list(range(count))) - self.assertEquals(self.q.elements(), [count - i - 1 for i in range(count)]) + self.assertEqual(self.q.elements(), [count - i - 1 for i in range(count)]) with tempfile.TemporaryFile() as f: self.q.dump(f) # Now, assert that it is empty - self.assertEquals(len(self.q), 0) + self.assertEqual(len(self.q), 0) # Now, try to load it back in f.seek(0) self.q.load(f) - self.assertEquals(len(self.q), count) - self.assertEquals(self.q.elements(), [count - i - 1 for i in range(count)]) + self.assertEqual(len(self.q), count) + self.assertEqual(self.q.elements(), [count - i - 1 for i in range(count)]) # Now clean up after myself f.truncate() self.q.clear() @@ -106,48 +106,48 @@ class CappedCollection(unittest.TestCase): def setUp(self): r.delete('qrtestcc') self.aq = qr.CappedCollection(key='qrtestcc', size=3) - self.assertEquals(len(self.aq), 0) + self.assertEqual(len(self.aq), 0) def test_roundtrip(self): aq = self.aq aq.push('foo') - self.assertEquals(len(aq), 1) - self.assertEquals(aq.pop(), 'foo') - self.assertEquals(len(aq), 0) + self.assertEqual(len(aq), 1) + self.assertEqual(aq.pop(), 'foo') + self.assertEqual(len(aq), 0) def test_order(self): aq = self.aq aq.push('foo') aq.push('bar') - self.assertEquals(aq.pop(), 'foo') - self.assertEquals(aq.pop(), 'bar') + self.assertEqual(aq.pop(), 'foo') + self.assertEqual(aq.pop(), 'bar') def test_order_mixed(self): aq = self.aq aq.push('foo') - self.assertEquals(aq.pop(), 'foo') + self.assertEqual(aq.pop(), 'foo') aq.push('bar') - self.assertEquals(aq.pop(), 'bar') + self.assertEqual(aq.pop(), 'bar') def test_limit(self): aq = self.aq aq.push('a') aq.push('b') aq.push('c') - self.assertEquals(len(aq), 3) + self.assertEqual(len(aq), 3) aq.push('d') aq.push('e') - self.assertEquals(len(aq), 3) - self.assertEquals(aq.pop(), 'c') - self.assertEquals(aq.pop(), 'd') - self.assertEquals(aq.pop(), 'e') - self.assertEquals(len(aq), 0) + self.assertEqual(len(aq), 3) + self.assertEqual(aq.pop(), 'c') + self.assertEqual(aq.pop(), 'd') + self.assertEqual(aq.pop(), 'e') + self.assertEqual(len(aq), 0) def test_extend(self): '''Test extending a queue, including with a generator''' count = 100 self.aq.extend(i for i in range(count)) - self.assertEquals(len(self.aq), self.aq.size) + self.assertEqual(len(self.aq), self.aq.size) self.aq.clear() class Stack(unittest.TestCase): @@ -158,23 +158,23 @@ def setUp(self): def test_roundtrip(self): stack = self.stack stack.push('foo') - self.assertEquals(len(stack), 1) - self.assertEquals(stack.pop(), 'foo') - self.assertEquals(len(stack), 0) + self.assertEqual(len(stack), 1) + self.assertEqual(stack.pop(), 'foo') + self.assertEqual(len(stack), 0) def test_order(self): stack = self.stack stack.push('foo') stack.push('bar') - self.assertEquals(stack.pop(), 'bar') - self.assertEquals(stack.pop(), 'foo') + self.assertEqual(stack.pop(), 'bar') + self.assertEqual(stack.pop(), 'foo') def test_order_mixed(self): stack = self.stack stack.push('foo') - self.assertEquals(stack.pop(), 'foo') + self.assertEqual(stack.pop(), 'foo') stack.push('bar') - self.assertEquals(stack.pop(), 'bar') + self.assertEqual(stack.pop(), 'bar') def test_get_item(self): count = 100 @@ -183,19 +183,19 @@ def test_get_item(self): items.reverse() # Get single values for i in range(count): - self.assertEquals(self.stack[i], items[i]) + self.assertEqual(self.stack[i], items[i]) # Get small ranges for i in range(count-1): - self.assertEquals(self.stack[i:i+2], items[i:i+2]) + self.assertEqual(self.stack[i:i+2], items[i:i+2]) # Now get the whole range - self.assertEquals(self.stack[0:-1], items[0:-1]) + self.assertEqual(self.stack[0:-1], items[0:-1]) self.stack.clear() def test_extend(self): '''Test extending a queue, including with a generator''' count = 100 self.stack.extend(i for i in range(count)) - self.assertEquals(self.stack.elements(), [count - i - 1 for i in range(count)]) + self.assertEqual(self.stack.elements(), [count - i - 1 for i in range(count)]) # Also, make sure it's still a stack. It should be in reverse order last = self.stack.pop() @@ -211,16 +211,16 @@ def test_dump_load(self): # Get a temporary file to dump a queue to that file count = 100 self.stack.extend(list(range(count))) - self.assertEquals(self.stack.elements(), [count - i - 1 for i in range(count)]) + self.assertEqual(self.stack.elements(), [count - i - 1 for i in range(count)]) with tempfile.TemporaryFile() as f: self.stack.dump(f) # Now, assert that it is empty - self.assertEquals(len(self.stack), 0) + self.assertEqual(len(self.stack), 0) # Now, try to load it back in f.seek(0) self.stack.load(f) - self.assertEquals(len(self.stack), count) - self.assertEquals(self.stack.elements(), [count - i - 1 for i in range(count)]) + self.assertEqual(len(self.stack), count) + self.assertEqual(self.stack.elements(), [count - i - 1 for i in range(count)]) # Now clean up after myself f.truncate() self.stack.clear() @@ -232,15 +232,15 @@ def setUp(self): def test_roundtrip(self): self.q.push('foo', 1) - self.assertEquals(len(self.q), 1) - self.assertEquals(self.q.pop(), 'foo') - self.assertEquals(len(self.q), 0) + self.assertEqual(len(self.q), 1) + self.assertEqual(self.q.pop(), 'foo') + self.assertEqual(len(self.q), 0) def test_order(self): self.q.push('foo', 1) self.q.push('bar', 0) - self.assertEquals(self.q.pop(), 'bar') - self.assertEquals(self.q.pop(), 'foo') + self.assertEqual(self.q.pop(), 'bar') + self.assertEqual(self.q.pop(), 'foo') def test_get_item(self): count = 100 @@ -248,12 +248,12 @@ def test_get_item(self): self.q.extend(list(zip(items, items))) # Get single values for i in range(count): - self.assertEquals(self.q[i], items[i]) + self.assertEqual(self.q[i], items[i]) # Get small ranges for i in range(count-1): - self.assertEquals(self.q[i:i+2], items[i:i+2]) + self.assertEqual(self.q[i:i+2], items[i:i+2]) # Now get the whole range - self.assertEquals(self.q[0:-1], items[0:-1]) + self.assertEqual(self.q[0:-1], items[0:-1]) self.q.clear() def test_extend(self): @@ -261,7 +261,7 @@ def test_extend(self): count = 100 items = [i for i in range(count)] self.q.extend(list(zip(items, items))) - self.assertEquals(self.q.elements(), items) + self.assertEqual(self.q.elements(), items) self.q.clear() def test_pop(self): @@ -297,7 +297,7 @@ def test_uniqueness(self): # Push the same value on with different scores for i in range(count): self.q.push(1, i) - self.assertEquals(len(self.q), 1) + self.assertEqual(len(self.q), 1) self.q.clear() def test_dump_load(self): @@ -305,16 +305,16 @@ def test_dump_load(self): count = 100 items = [i for i in range(count)] self.q.extend(list(zip(items, items))) - self.assertEquals(self.q.elements(), items) + self.assertEqual(self.q.elements(), items) with tempfile.TemporaryFile() as f: self.q.dump(f) # Now, assert that it is empty - self.assertEquals(len(self.q), 0) + self.assertEqual(len(self.q), 0) # Now, try to load it back in f.seek(0) self.q.load(f) - self.assertEquals(len(self.q), count) - self.assertEquals(self.q.elements(), items) + self.assertEqual(len(self.q), count) + self.assertEqual(self.q.elements(), items) # Now clean up after myself f.truncate() self.q.clear() From 5213fa575701d0c6159c554dd17bc28b24acdc21 Mon Sep 17 00:00:00 2001 From: Vinay Valsaraj Date: Fri, 6 Jul 2018 16:18:41 -0700 Subject: [PATCH 06/13] Updated setup.py file for dependencies needed (future and redis) --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5e2152a..9af580b 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -import os -import unittest from setuptools import setup, find_packages version = '0.6.0' @@ -39,6 +37,7 @@ keywords = 'Redis, queue, data structures', license = 'MIT', packages = find_packages(), + install_requires = ['future', 'redis'], py_modules = ['qr'], include_package_data = True, zip_safe = False, From 735a28729a35c176d0f867522b32baf3ec6c9e51 Mon Sep 17 00:00:00 2001 From: Vinay Valsaraj Date: Tue, 17 Jul 2018 11:08:23 -0700 Subject: [PATCH 07/13] Updated all information in setup, followed PEP8 spacing, and refactored imports --- qr.py | 24 +++++++++++++++--------- setup.py | 23 +++++++++++++---------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/qr.py b/qr.py index 294aae9..f305ac0 100644 --- a/qr.py +++ b/qr.py @@ -6,11 +6,13 @@ standard_library.install_aliases() from builtins import object __author__ = 'Ted Nyman' -__version__ = '0.6.0' +__version__ = '1.0.0' __license__ = 'MIT' -import redis import logging +import pickle +import redis + try: import json @@ -20,10 +22,7 @@ # This is a complete nod to hotqueue -- this is one of the # things that they did right. Natively pickling and unpiclking # objects is pretty useful. -try: - import pickle as pickle -except ImportError: - import pickle + class NullHandler(logging.Handler): """A logging handler that discards all logging records""" @@ -39,6 +38,7 @@ def emit(self, record): # connections connectionPools = {} + def getRedis(**kwargs): """ Match up the provided kwargs with an existing connection pool. @@ -55,6 +55,7 @@ def getRedis(**kwargs): connectionPools[key] = cp return redis.Redis(connection_pool=cp) + class worker(object): def __init__(self, q, err=None, *args, **kwargs): self.q = q @@ -81,7 +82,8 @@ def wrapped(): except: pass return wrapped - + + class BaseQueue(object): """Base functionality common to queues""" @staticmethod @@ -93,7 +95,7 @@ def __init__(self, key, **kwargs): self.serializer = pickle self.redis = getRedis(**kwargs) self.key = key - + def __len__(self): """Return the length of the queue""" return self.redis.llen(self.key) @@ -201,6 +203,7 @@ def pop_back(self): log.debug('Popped ** %s ** from key ** %s **' % (popped, self.key)) return self._unpack(popped) + class Queue(BaseQueue): """Implements a FIFO queue""" @@ -221,7 +224,8 @@ def pop(self, block=False): queue, popped = self.redis.brpop(self.key) log.debug('Popped ** %s ** from key ** %s **' % (popped, self.key)) return self._unpack(popped) - + + class PriorityQueue(BaseQueue): """A priority queue""" def __len__(self): @@ -315,6 +319,7 @@ def push(self, value, score): '''Add an element with a given score''' return self.redis.zadd(self.key, self._pack(value), score) + class CappedCollection(BaseQueue): """ Implements a capped collection (the collection never @@ -352,6 +357,7 @@ def pop(self, block=False): log.debug('Popped ** %s ** from key ** %s **' % (popped, self.key)) return self._unpack(popped) + class Stack(BaseQueue): """Implements a LIFO stack""" diff --git a/setup.py b/setup.py index 9af580b..bd33a50 100644 --- a/setup.py +++ b/setup.py @@ -2,18 +2,18 @@ from setuptools import setup, find_packages -version = '0.6.0' +version = '1.0.0' LONG_DESCRIPTION = ''' -Full documentation (with example code) is at http://github.com/tnm/qr +Full documentation (with example code) is at http://github.com/doctorondemand/qr3 -QR +QR3 ===== -**QR** helps you create and work with **queue, capped collection (bounded queue), -deque, and stack** data structures for **Redis**. Redis is well-suited for -implementations of these abstract data structures, and QR makes it even easier to +**QR3** helps you create and work with **queue, capped collection (bounded queue), +deque, and stack** data structures for **Redis**. Redis is well-suited for +implementations of these abstract data structures, and QR3 makes it even easier to work with the structures in Python. Quick Setup @@ -22,18 +22,20 @@ of MULTI/EXEC, so you'll need the Git edge version), and the current Python interface for Redis, [redis-py](http://github.com/andymccurdy/redis-py "redis-py"). -Run setup.py to install qr. +Run setup.py to install qr3 or 'pip install qr3'. ''' setup( - name = 'qr', + name = 'qr3', version = version, description = 'Redis-powered queues, capped collections, deques, and stacks', long_description = LONG_DESCRIPTION, - url = 'http://github.com/tnm/qr', + url = 'http://github.com/doctorondemand/qr3', author = 'Ted Nyman', author_email = 'ted@ted.io', + maintainer = 'DoctorOnDemand', + maintainer_email = 'dev@doctorondemand.com', keywords = 'Redis, queue, data structures', license = 'MIT', packages = find_packages(), @@ -42,7 +44,8 @@ include_package_data = True, zip_safe = False, classifiers = [ - 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', From d6caa7fd45bc5f3c7e4cab4c461fca17e07ca9ba Mon Sep 17 00:00:00 2001 From: Vinay Valsaraj Date: Tue, 17 Jul 2018 11:43:38 -0700 Subject: [PATCH 08/13] Created new package for qr3 and refactored tests to be run with unittest. --- qr3/__init__.py | 1 + qr.py => qr3/qr.py | 0 test/{tests.py => test_qr3.py} | 24 +++++++++++++----------- 3 files changed, 14 insertions(+), 11 deletions(-) create mode 100644 qr3/__init__.py rename qr.py => qr3/qr.py (100%) rename test/{tests.py => test_qr3.py} (99%) diff --git a/qr3/__init__.py b/qr3/__init__.py new file mode 100644 index 0000000..6d61f01 --- /dev/null +++ b/qr3/__init__.py @@ -0,0 +1 @@ +"""This package contains the qr classes.""" diff --git a/qr.py b/qr3/qr.py similarity index 100% rename from qr.py rename to qr3/qr.py diff --git a/test/tests.py b/test/test_qr3.py similarity index 99% rename from test/tests.py rename to test/test_qr3.py index 331121f..6a296b5 100644 --- a/test/tests.py +++ b/test/test_qr3.py @@ -1,12 +1,13 @@ from builtins import zip from builtins import range -import qr +from qr3 import qr import redis import tempfile import unittest r = redis.Redis() + class Queue(unittest.TestCase): def setUp(self): r.delete('qrtestqueue') @@ -59,22 +60,22 @@ def test_get_item(self): # Now get the whole range self.assertEqual(self.q[0:-1], items[0:-1]) self.q.clear() - + def test_extend(self): '''Test extending a queue, including with a generator''' count = 100 self.q.extend(i for i in range(count)) self.assertEqual(len(self.q), count) self.q.clear() - + self.q.extend([i for i in range(count)]) self.assertEqual(len(self.q), count) self.q.clear() - + self.q.extend(list(range(count))) self.assertEqual(self.q.elements(), [count - i - 1 for i in range(count)]) self.q.clear() - + def test_pack_unpack(self): '''Make sure that it behaves like python-object-in, python-object-out''' count = 100 @@ -83,7 +84,7 @@ def test_pack_unpack(self): while next: self.assertTrue(isinstance(next, dict)) next = self.q.pop() - + def test_dump_load(self): # Get a temporary file to dump a queue to that file count = 100 @@ -101,7 +102,7 @@ def test_dump_load(self): # Now clean up after myself f.truncate() self.q.clear() - + class CappedCollection(unittest.TestCase): def setUp(self): r.delete('qrtestcc') @@ -196,7 +197,7 @@ def test_extend(self): count = 100 self.stack.extend(i for i in range(count)) self.assertEqual(self.stack.elements(), [count - i - 1 for i in range(count)]) - + # Also, make sure it's still a stack. It should be in reverse order last = self.stack.pop() now = None @@ -281,7 +282,7 @@ def test_pop(self): self.assertTrue(isinstance(value, int)) self.assertTrue(isinstance(score, float)) value, score = self.q.pop(withscores=True) - + def test_push(self): '''Test whether we can push well''' count = 100 @@ -291,7 +292,7 @@ def test_push(self): while value: self.assertEqual(value + score, count) value, score = self.q.pop(withscores=True) - + def test_uniqueness(self): count = 100 # Push the same value on with different scores @@ -299,7 +300,7 @@ def test_uniqueness(self): self.q.push(1, i) self.assertEqual(len(self.q), 1) self.q.clear() - + def test_dump_load(self): # Get a temporary file to dump a queue to that file count = 100 @@ -319,5 +320,6 @@ def test_dump_load(self): f.truncate() self.q.clear() + if __name__ == '__main__': unittest.main() From 2171c01568b5ee4f6acb9c0206d4ade199f62b96 Mon Sep 17 00:00:00 2001 From: Vinay Valsaraj Date: Mon, 23 Jul 2018 14:48:38 -0700 Subject: [PATCH 09/13] Updated setup.py and README with responding to PR's --- README.md | 9 +++++++-- setup.py | 9 +++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e585d78..552bb34 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ -QR +QR3 ===== -**QR** helps you create and work with **queue, capped collection (bounded queue), deque, and stack** data structures for **Redis**. +**QR3** helps you create and work with **queue, capped collection (bounded queue), deque, and stack** data structures for **Redis**. Redis is well-suited for implementations of these abstract data structures, and QR makes it even easier to work with the structures in Python. Quick Setup @@ -25,6 +25,11 @@ Then install `qr`: python setup.py install ``` +To run tests: +``` +python -m unittest discover -v +``` + Basics of QR ------------------ diff --git a/setup.py b/setup.py index bd33a50..cd03465 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,15 @@ Run setup.py to install qr3 or 'pip install qr3'. +Given that this package primarily supports internal use cases, we cannot guarantee a +specific response time on PRs for new features. However, we will do our best to +consider them in a timely fashion. + +We do commit to reviewing anything related to a security issue in a timely manner. +We ask that you first submit anything of that nature to security@doctorondemand.com +prior to creating a PR and follow responsible disclosure rules. + +Thanks for your interest in helping with this package! ''' setup( From de4bf3c3e4e6bb962a9742ca392b559967842bbf Mon Sep 17 00:00:00 2001 From: Vinay Valsaraj Date: Mon, 23 Jul 2018 14:48:38 -0700 Subject: [PATCH 10/13] Updated setup.py and README with responding to PR's --- README.md | 21 +++++++++++++++++++-- setup.py | 13 ++++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e585d78..807d35e 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ -QR +QR3 ===== -**QR** helps you create and work with **queue, capped collection (bounded queue), deque, and stack** data structures for **Redis**. +**QR3** helps you create and work with **queue, capped collection (bounded queue), deque, and stack** data structures for **Redis**. Redis is well-suited for implementations of these abstract data structures, and QR makes it even easier to work with the structures in Python. Quick Setup @@ -25,6 +25,23 @@ Then install `qr`: python setup.py install ``` +To run tests: +``` +python -m unittest discover -v +``` + +Responding to PR's +------------------ +Given that this package primarily supports internal use cases, we cannot guarantee a +specific response time on PRs for new features. However, we will do our best to +consider them in a timely fashion. + +We do commit to reviewing anything related to a security issue in a timely manner. +We ask that you first submit anything of that nature to security@doctorondemand.com +prior to creating a PR and follow responsible disclosure rules. + +Thanks for your interest in helping with this package! + Basics of QR ------------------ diff --git a/setup.py b/setup.py index bd33a50..fd5f5a0 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,17 @@ Run setup.py to install qr3 or 'pip install qr3'. +Responding to PR's +------------------ +Given that this package primarily supports internal use cases, we cannot guarantee a +specific response time on PRs for new features. However, we will do our best to +consider them in a timely fashion. + +We do commit to reviewing anything related to a security issue in a timely manner. +We ask that you first submit anything of that nature to security@doctorondemand.com +prior to creating a PR and follow responsible disclosure rules. + +Thanks for your interest in helping with this package! ''' setup( @@ -35,7 +46,7 @@ author = 'Ted Nyman', author_email = 'ted@ted.io', maintainer = 'DoctorOnDemand', - maintainer_email = 'dev@doctorondemand.com', + maintainer_email = 'sustaining@doctorondemand.com', keywords = 'Redis, queue, data structures', license = 'MIT', packages = find_packages(), From 613c1faffb30aa3ca8ca807b2aaaab26ebb2c447 Mon Sep 17 00:00:00 2001 From: Adam King Date: Thu, 30 Jan 2020 17:20:32 -0600 Subject: [PATCH 11/13] Update for redis-py ver 3+ compatibility --- qr3/qr.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/qr3/qr.py b/qr3/qr.py index f305ac0..51d529a 100644 --- a/qr3/qr.py +++ b/qr3/qr.py @@ -1,4 +1,5 @@ """ + QR | Redis-Based Data Structures in Python """ @@ -10,6 +11,7 @@ __license__ = 'MIT' import logging +from packaging import version import pickle import redis @@ -94,6 +96,8 @@ def all(t, pattern, **kwargs): def __init__(self, key, **kwargs): self.serializer = pickle self.redis = getRedis(**kwargs) + # redis-py version 3 handles the 'zadd' functionality differently than it did for v2. + self.redis_version_3_or_greater = version.parse(redis.__version__) >= version.parse("3.0") self.key = key def __len__(self): @@ -279,7 +283,10 @@ def extend(self, vals): """Extends the elements in the queue.""" with self.redis.pipeline(transaction=False) as pipe: for val, score in vals: - pipe.zadd(self.key, self._pack(val), score) + if self.redis_version_3_or_greater: + pipe.zadd(self.key, {self._pack(val): score}) + else: + pipe.zadd(self.key, self._pack(val), score) return pipe.execute() def peek(self, withscores=False): @@ -317,7 +324,10 @@ def pop(self, withscores=False): def push(self, value, score): '''Add an element with a given score''' - return self.redis.zadd(self.key, self._pack(value), score) + if self.redis_version_3_or_greater: + return self.redis.zadd(self.key, {self._pack(value): score}) + else: + return self.redis.zadd(self.key, self._pack(value), score) class CappedCollection(BaseQueue): From 9fed24fb2ce39f858488b84a807833779538b26b Mon Sep 17 00:00:00 2001 From: Adam King Date: Fri, 31 Jan 2020 15:19:27 -0600 Subject: [PATCH 12/13] Add requirement for redis v 3 or greater --- qr3/qr.py | 13 ++----------- setup.py | 2 +- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/qr3/qr.py b/qr3/qr.py index 51d529a..5c52d52 100644 --- a/qr3/qr.py +++ b/qr3/qr.py @@ -11,7 +11,6 @@ __license__ = 'MIT' import logging -from packaging import version import pickle import redis @@ -96,8 +95,6 @@ def all(t, pattern, **kwargs): def __init__(self, key, **kwargs): self.serializer = pickle self.redis = getRedis(**kwargs) - # redis-py version 3 handles the 'zadd' functionality differently than it did for v2. - self.redis_version_3_or_greater = version.parse(redis.__version__) >= version.parse("3.0") self.key = key def __len__(self): @@ -283,10 +280,7 @@ def extend(self, vals): """Extends the elements in the queue.""" with self.redis.pipeline(transaction=False) as pipe: for val, score in vals: - if self.redis_version_3_or_greater: - pipe.zadd(self.key, {self._pack(val): score}) - else: - pipe.zadd(self.key, self._pack(val), score) + pipe.zadd(self.key, {self._pack(val): score}) return pipe.execute() def peek(self, withscores=False): @@ -324,10 +318,7 @@ def pop(self, withscores=False): def push(self, value, score): '''Add an element with a given score''' - if self.redis_version_3_or_greater: - return self.redis.zadd(self.key, {self._pack(value): score}) - else: - return self.redis.zadd(self.key, self._pack(value), score) + return self.redis.zadd(self.key, {self._pack(value): score}) class CappedCollection(BaseQueue): diff --git a/setup.py b/setup.py index fd5f5a0..a73e375 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ keywords = 'Redis, queue, data structures', license = 'MIT', packages = find_packages(), - install_requires = ['future', 'redis'], + install_requires = ['future', 'redis>=3.0.0'], py_modules = ['qr'], include_package_data = True, zip_safe = False, From 9979c44057b6d53f3a9b6bef30bbb098ff44e33d Mon Sep 17 00:00:00 2001 From: Adam King Date: Fri, 31 Jan 2020 16:41:10 -0600 Subject: [PATCH 13/13] update version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a73e375..680cb7f 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages -version = '1.0.0' +version = '1.0.1' LONG_DESCRIPTION = '''