Merge pull request #163 from Digenis/deprecate_excess_sqlite_utils

Deprecate alternative sqlite backed queues/dictionaries

docs/news.rst

@@ -11,6 +11,16 @@ The highlight of this release is the long-awaited Python 3 support.
 The new scrapy requirement is version 1.0 or higher.
 Python 2.6 is no longer supported by scrapyd.
 
+Some unused sqlite utilities in are now deprecated
+and will be removed from a later scrapyd release.
+Instantiating them or subclassing from them
+will trigger a deprecation warning.
+These are located under ``scrapyd.sqlite``:
+- SqliteDict
+- SqlitePickleDict
+- SqlitePriorityQueue
+- PickleSqlitePriorityQueue
+
 Added
 ~~~~~
 

scrapyd/_deprecate.py

@@ -0,0 +1,38 @@
+import warnings
+import inspect
+
+
+class ScrapydDeprecationWarning(Warning):
+    """Warning category for deprecated features, since the default
+    DeprecationWarning is silenced on Python 2.7+
+    """
+    pass
+
+
+class WarningMeta(type):
+    def __init__(cls, name, bases, clsdict):
+        offending_wrapper_classes = tuple(c.__bases__ for c in bases
+                                          if isinstance(c, WarningMeta))
+        offending_classes = tuple(c for c, in offending_wrapper_classes)
+        if offending_classes:
+            warnings.warn(
+                '%r inherits from %r which %s deprecated'
+                ' and will be removed from a later scrapyd release'
+                % (cls, offending_classes,
+                   ['is', 'are'][min(2, len(offending_classes))-1]),
+                ScrapydDeprecationWarning,
+            )
+        super(WarningMeta, cls).__init__(name, bases, clsdict)
+
+
+def deprecate_class(cls):
+    class WarningMeta2(WarningMeta):
+        pass
+    for b in cls.__bases__:
+        if type(b) not in WarningMeta2.__bases__:
+            WarningMeta2.__bases__ += (type(b),)
+    def new_init(*args, **kwargs):
+        warnings.warn('%r will be removed from a later scrapyd release' % cls,
+                      ScrapydDeprecationWarning)
+        return cls.__init__(*args, **kwargs)
+    return WarningMeta2(cls.__name__, (cls,), {'__init__': new_init})

scrapyd/sqlite.py

@@ -11,7 +11,10 @@
 import six
 
 
-class SqliteDict(MutableMapping):
+from ._deprecate import deprecate_class
+
+
+class JsonSqliteDict(MutableMapping):
     """SQLite-backed dictionary"""
 
     def __init__(self, database=None, table="dict"):
@@ -73,13 +76,14 @@ def items(self):
         return list(self.iteritems())
 
     def encode(self, obj):
-        return obj
+        return sqlite3.Binary(json.dumps(obj).encode('ascii'))
 
     def decode(self, obj):
-        return obj
+        return json.loads(bytes(obj).decode('ascii'))
 
 
-class PickleSqliteDict(SqliteDict):
+@deprecate_class
+class PickleSqliteDict(JsonSqliteDict):
 
     def encode(self, obj):
         return sqlite3.Binary(pickle.dumps(obj, protocol=2))
@@ -88,16 +92,17 @@ def decode(self, obj):
         return pickle.loads(bytes(obj))
 
 
-class JsonSqliteDict(SqliteDict):
+@deprecate_class
+class SqliteDict(JsonSqliteDict):
 
     def encode(self, obj):
-        return sqlite3.Binary(json.dumps(obj).encode('ascii'))
+        return obj
 
     def decode(self, obj):
-        return json.loads(bytes(obj).decode('ascii'))
+        return obj
 
 
-class SqlitePriorityQueue(object):
+class JsonSqlitePriorityQueue(object):
     """SQLite priority queue. It relies on SQLite concurrency support for
     providing atomic inter-process operations.
     """
@@ -160,13 +165,14 @@ def __iter__(self):
         return ((self.decode(x), y) for x, y in self.conn.execute(q))
 
     def encode(self, obj):
-        return obj
+        return sqlite3.Binary(json.dumps(obj).encode('ascii'))
 
     def decode(self, text):
-        return text
+        return json.loads(bytes(text).decode('ascii'))
 
 
-class PickleSqlitePriorityQueue(SqlitePriorityQueue):
+@deprecate_class
+class PickleSqlitePriorityQueue(JsonSqlitePriorityQueue):
 
     def encode(self, obj):
         return sqlite3.Binary(pickle.dumps(obj, protocol=2))
@@ -175,10 +181,11 @@ def decode(self, obj):
         return pickle.loads(bytes(obj))
 
 
-class JsonSqlitePriorityQueue(SqlitePriorityQueue):
+@deprecate_class
+class SqlitePriorityQueue(JsonSqlitePriorityQueue):
 
     def encode(self, obj):
-        return sqlite3.Binary(json.dumps(obj).encode('ascii'))
+        return obj
 
     def decode(self, obj):
-        return json.loads(bytes(obj).decode('ascii'))
+        return obj