Py2/3 Compatibility and Bug Fixes in dump/load for PriorityQueue

README.md

@@ -1,7 +1,7 @@
-QR
+QR3
 =====
 
-**QR** helps you create and work with **queue, capped collection (bounded queue), deque, and stack** data structures for **Redis**. 
+**QR3** helps you create and work with **queue, capped collection (bounded queue), deque, and stack** data structures for **Redis**.
 Redis is well-suited for implementations of these abstract data structures, and QR makes it even easier to work with the structures in Python.
 
 Quick Setup
@@ -25,6 +25,23 @@ Then install `qr`:
 python setup.py install
 ```
 
+To run tests:
+```
+python -m unittest discover -v
+```
+
+Responding to PR's
+------------------
+Given that this package primarily supports internal use cases, we cannot guarantee a
+specific response time on PRs for new features. However, we will do our best to
+consider them in a timely fashion.
+
+We do commit to reviewing anything related to a security issue in a timely manner.
+We ask that you first submit anything of that nature to security@doctorondemand.com
+prior to creating a PR and follow responsible disclosure rules.
+
+Thanks for your interest in helping with this package!
+
 Basics of QR
 ------------------
 

qr3/__init__.py

@@ -0,0 +1 @@
+"""This package contains the qr classes."""

qr.py → qr3/qr.py

@@ -1,13 +1,19 @@
 """
+
 QR | Redis-Based Data Structures in Python
 """
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import object
 __author__ = 'Ted Nyman'
-__version__ = '0.6.0'
+__version__ = '1.0.0'
 __license__ = 'MIT'
 
-import redis
 import logging
+import pickle
+import redis
+
 
 try:
     import json
@@ -17,10 +23,7 @@
 # This is a complete nod to hotqueue -- this is one of the
 # things that they did right. Natively pickling and unpiclking
 # objects is pretty useful.
-try:
-    import cPickle as pickle
-except ImportError:
-    import pickle
+
 
 class NullHandler(logging.Handler):
     """A logging handler that discards all logging records"""
@@ -36,6 +39,7 @@ def emit(self, record):
 # connections
 connectionPools = {}
 
+
 def getRedis(**kwargs):
     """
     Match up the provided kwargs with an existing connection pool.
@@ -44,14 +48,15 @@ def getRedis(**kwargs):
     connection pool mechanism to keep the number of open file descriptors
     tractable.
     """
-    key = ':'.join((repr(key) + '=>' + repr(value)) for key, value in kwargs.items())
+    key = ':'.join((repr(key) + '=>' + repr(value)) for key, value in list(kwargs.items()))
     try:
         return redis.Redis(connection_pool=connectionPools[key])
     except KeyError:
         cp = redis.ConnectionPool(**kwargs)
         connectionPools[key] = cp
         return redis.Redis(connection_pool=cp)
 
+
 class worker(object):
     def __init__(self, q, err=None, *args, **kwargs):
         self.q = q
@@ -78,7 +83,8 @@ def wrapped():
                     except:
                         pass
         return wrapped
-
+
+
 class BaseQueue(object):
     """Base functionality common to queues"""
     @staticmethod
@@ -90,7 +96,7 @@ def __init__(self, key, **kwargs):
         self.serializer = pickle
         self.redis = getRedis(**kwargs)
         self.key = key
-    
+
     def __len__(self):
         """Return the length of the queue"""
         return self.redis.llen(self.key)
@@ -134,15 +140,15 @@ def load(self, fobj):
     def dumpfname(self, fname, truncate=False):
         """Destructively dump the contents of the queue into fname"""
         if truncate:
-            with file(fname, 'w+') as f:
+            with open(fname, 'w+') as f:
                 self.dump(f)
         else:
-            with file(fname, 'a+') as f:
+            with open(fname, 'a+') as f:
                 self.dump(f)
 
     def loadfname(self, fname):
         """Load the contents of the contents of fname into the queue"""
-        with file(fname) as f:
+        with open(fname) as f:
             self.load(f)
 
     def extend(self, vals):
@@ -198,6 +204,7 @@ def pop_back(self):
         log.debug('Popped ** %s ** from key ** %s **' % (popped, self.key))
         return self._unpack(popped)
 
+
 class Queue(BaseQueue): 
     """Implements a FIFO queue"""
 
@@ -218,7 +225,8 @@ def pop(self, block=False):
             queue, popped = self.redis.brpop(self.key)
         log.debug('Popped ** %s ** from key ** %s **' % (popped, self.key))
         return self._unpack(popped)
-
+
+
 class PriorityQueue(BaseQueue):
     """A priority queue"""
     def __len__(self):
@@ -240,39 +248,39 @@ def __getitem__(self, val):
 
     def dump(self, fobj):
         """Destructively dump the contents of the queue into fp"""
-        next = self.pop()
-        while next:
-            self.serializer.dump(next[0], fobj)
-            next = self.pop()
-    
+        next = self.pop(True)
+        while next[0] is not None:
+            self.serializer.dump(next, fobj)
+            next = self.pop(True)
+
     def load(self, fobj):
         """Load the contents of the provided fobj into the queue"""
         try:
             while True:
                 value, score = self.serializer.load(fobj)
-                self.redis.zadd(self.key, value, score)
+                self.push(value, score)
         except Exception as e:
             return
 
     def dumpfname(self, fname, truncate=False):
         """Destructively dump the contents of the queue into fname"""
         if truncate:
-            with file(fname, 'w+') as f:
+            with open(fname, 'w+') as f:
                 self.dump(f)
         else:
-            with file(fname, 'a+') as f:
+            with open(fname, 'a+') as f:
                 self.dump(f)
 
     def loadfname(self, fname):
         """Load the contents of the contents of fname into the queue"""
-        with file(fname) as f:
+        with open(fname) as f:
             self.load(f)
 
     def extend(self, vals):
         """Extends the elements in the queue."""
         with self.redis.pipeline(transaction=False) as pipe:
             for val, score in vals:
-                pipe.zadd(self.key, self._pack(val), score)
+                pipe.zadd(self.key, {self._pack(val): score})
             return pipe.execute()
 
     def peek(self, withscores=False):
@@ -310,7 +318,8 @@ def pop(self, withscores=False):
 
     def push(self, value, score):
         '''Add an element with a given score'''
-        return self.redis.zadd(self.key, self._pack(value), score)
+        return self.redis.zadd(self.key, {self._pack(value): score})
+
 
 class CappedCollection(BaseQueue):
     """
@@ -349,6 +358,7 @@ def pop(self, block=False):
         log.debug('Popped ** %s ** from key ** %s **' % (popped, self.key))
         return self._unpack(popped)
 
+
 class Stack(BaseQueue):
     """Implements a LIFO stack""" 
 

setup.py

@@ -1,21 +1,19 @@
 #!/usr/bin/env python
 
-import os
-import unittest
 from setuptools import setup, find_packages
 
-version = '0.6.0'
+version = '1.0.1'
 
 LONG_DESCRIPTION = '''
 
-Full documentation (with example code) is at http://github.com/tnm/qr
+Full documentation (with example code) is at http://github.com/doctorondemand/qr3
 
-QR
+QR3
 =====
 
-**QR** helps you create and work with **queue, capped collection (bounded queue), 
-deque, and stack** data structures for **Redis**. Redis is well-suited for 
-implementations of these abstract data structures, and QR makes it even easier to 
+**QR3** helps you create and work with **queue, capped collection (bounded queue),
+deque, and stack** data structures for **Redis**. Redis is well-suited for
+implementations of these abstract data structures, and QR3 makes it even easier to
 work with the structures in Python.
 
 Quick Setup
@@ -24,26 +22,41 @@
 of MULTI/EXEC, so you'll need the Git edge version), and the current Python interface
 for Redis, [redis-py](http://github.com/andymccurdy/redis-py "redis-py"). 
 
-Run setup.py to install qr.
+Run setup.py to install qr3 or 'pip install qr3'.
 
+Responding to PR's
+------------------
+Given that this package primarily supports internal use cases, we cannot guarantee a 
+specific response time on PRs for new features. However, we will do our best to 
+consider them in a timely fashion.
+
+We do commit to reviewing anything related to a security issue in a timely manner. 
+We ask that you first submit anything of that nature to security@doctorondemand.com 
+prior to creating a PR and follow responsible disclosure rules.
+
+Thanks for your interest in helping with this package!
 '''
 
 setup(
-    name                 = 'qr',
+    name                 = 'qr3',
     version              = version,
     description          = 'Redis-powered queues, capped collections, deques, and stacks',
     long_description     = LONG_DESCRIPTION,
-    url                  = 'http://github.com/tnm/qr',
+    url                  = 'http://github.com/doctorondemand/qr3',
     author               = 'Ted Nyman',
     author_email         = 'ted@ted.io',
+    maintainer           = 'DoctorOnDemand',
+    maintainer_email     = 'sustaining@doctorondemand.com',
     keywords             = 'Redis, queue, data structures',
     license              = 'MIT',
     packages             = find_packages(),
+    install_requires     = ['future', 'redis>=3.0.0'],
     py_modules           = ['qr'],
     include_package_data = True,
     zip_safe             = False,
     classifiers          = [
-	'Programming Language :: Python',
+	    'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3.6',
         'Intended Audience :: Developers',
         'License :: OSI Approved :: MIT License',
         'Operating System :: OS Independent',