a lot of bytes/unicode mangling and updating protobufs decoder

pybase/client.py

@@ -16,6 +16,7 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import logging
+from builtins import str
 from itertools import chain
 from threading import Lock
 
@@ -341,15 +342,15 @@ def _create_new_region(self, response, table):
         # We get ~4 cells back each holding different information. We only care
         # about two of them.
         for cell in cells:
-            if cell.qualifier == "regioninfo":
+            if cell.qualifier == b"regioninfo":
                 # Take the regioninfo information and parse it into our own
                 # Region representation.
                 new_region = region_from_cell(cell)
-            elif cell.qualifier == "server":
+            elif cell.qualifier == b"server":
                 # Grab the host, port of the Region Server that this region is
                 # hosted on.
                 server_loc = cell.value
-                host, port = cell.value.split(':')
+                host, port = cell.value.split(b':')
             else:
                 continue
         # Do we have an existing client for this region server already?
@@ -423,7 +424,11 @@ def _purge_region(self, reg):
                 pass
 
     def _construct_meta_key(self, table, key):
-        return table + b"," + key + b",:"
+        if isinstance(table, str):
+            table = table.encode('utf8')
+        if isinstance(key, str):
+            key = key.encode('utf8')
+        return b"%b,%b,:" % (table, key)
 
     def close(self):
         logger.info("Main client received close request.")

pybase/helpers/varint.py

@@ -29,77 +29,67 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 from __future__ import absolute_import, print_function, unicode_literals
 
+import six
+
 
 class NotEnoughDataExcption(Exception):
     pass
 
 
-def _VarintDecoder(mask):
+def _VarintDecoder(mask, result_type=int):
     """Return an encoder for a basic varint value (does not include tag).
-
-    Decoded values will be bitwise-anded with the given mask before being
-    returned, e.g. to limit them to 32 bits.  The returned decoder does not
-    take the usual "end" parameter -- the caller is expected to do bounds checking
-    after the fact (often the caller can defer such checking until later).  The
-    decoder returns a (value, new_pos) pair.
-    """
-
-    local_ord = ord
-
+      Decoded values will be bitwise-anded with the given mask before being
+      returned, e.g. to limit them to 32 bits.  The returned decoder does not
+      take the usual "end" parameter -- the caller is expected to do bounds checking
+      after the fact (often the caller can defer such checking until later).  The
+      decoder returns a (value, new_pos) pair.
+      """
     def DecodeVarint(buffer, pos):
         result = 0
         shift = 0
         while 1:
-            if pos > len(buffer) - 1:
-                raise NotEnoughDataExcption("Not enough data to decode varint")
-            b = local_ord(buffer[pos])
+            b = six.indexbytes(buffer, pos)
             result |= ((b & 0x7f) << shift)
             pos += 1
             if not (b & 0x80):
                 result &= mask
+                result = result_type(result)
                 return (result, pos)
             shift += 7
             if shift >= 64:
                 raise ValueError('Too many bytes when decoding varint.')
-
     return DecodeVarint
 
 
-def _SignedVarintDecoder(mask):
+def _SignedVarintDecoder(bits, result_type=int):
     """Like _VarintDecoder() but decodes signed values."""
-
-    local_ord = ord
+    signbit = 1 << (bits - 1)
+    mask = (1 << bits) - 1
 
     def DecodeVarint(buffer, pos):
         result = 0
         shift = 0
         while 1:
-            if pos > len(buffer) - 1:
-                raise NotEnoughDataExcption("Not enough data to decode varint")
-            b = local_ord(buffer[pos])
+            b = six.indexbytes(buffer, pos)
             result |= ((b & 0x7f) << shift)
             pos += 1
             if not (b & 0x80):
-                if result > 0x7fffffffffffffff:
-                    result -= (1 << 64)
-                    result |= ~mask
-                else:
-                    result &= mask
+                result &= mask
+                result = (result ^ signbit) - signbit
+                result = result_type(result)
                 return (result, pos)
             shift += 7
             if shift >= 64:
                 raise ValueError('Too many bytes when decoding varint.')
-
     return DecodeVarint
 
 
 decodeVarint = _VarintDecoder((1 << 64) - 1)
-decodeSignedVarint = _SignedVarintDecoder((1 << 64) - 1)
-
+decodeSignedVarint = _SignedVarintDecoder(64, int)
 
 # Use these versions for values which must be limited to 32 bits.
 decodeVarint32 = _VarintDecoder((1 << 32) - 1)
-decodeSignedVarint32 = _SignedVarintDecoder((1 << 32) - 1)
+decodeSignedVarint32 = _SignedVarintDecoder(32, int)
 
 
 def varintSize(value):
@@ -151,38 +141,31 @@ def signedVarintSize(value):
 
 
 def _VarintEncoder():
-    """Return an encoder for a basic varint value."""
-
-    local_chr = chr
-
-    def EncodeVarint(write, value):
+    """Return an encoder for a basic varint value (does not include tag)."""
+    def EncodeVarint(write, value, unused_deterministic=None):
         bits = value & 0x7f
         value >>= 7
         while value:
-            write(local_chr(0x80 | bits))
+            write(six.int2byte(0x80|bits))
             bits = value & 0x7f
             value >>= 7
-        return write(local_chr(bits))
-
+        return write(six.int2byte(bits))
     return EncodeVarint
 
 
 def _SignedVarintEncoder():
-    """Return an encoder for a basic signed varint value."""
-
-    local_chr = chr
-
-    def EncodeSignedVarint(write, value):
+    """Return an encoder for a basic signed varint value (does not include
+    tag)."""
+    def EncodeSignedVarint(write, value, unused_deterministic=None):
         if value < 0:
             value += (1 << 64)
         bits = value & 0x7f
         value >>= 7
         while value:
-            write(local_chr(0x80 | bits))
+            write(six.int2byte(0x80|bits))
             bits = value & 0x7f
             value >>= 7
-        return write(local_chr(bits))
-
+        return write(six.int2byte(bits))
     return EncodeSignedVarint
 
 

pybase/region/client.py

@@ -36,9 +36,9 @@
 # We need to know how to interpret an incoming proto.Message. This maps
 # the request_type to the response_type.
 response_types = {
-    "Get": GetResponse,
-    "Mutate": MutateResponse,
-    "Scan": ScanResponse
+    b"Get": GetResponse,
+    b"Mutate": MutateResponse,
+    b"Scan": ScanResponse
 }
 
 
@@ -291,4 +291,4 @@ def _send_hello(sock):
 def _to_varint(val):
     temp = []
     encoder(temp.append, val)
-    return b"".join(temp)
+    return "".join(temp).encode('utf8')

pybase/request/request.py

@@ -1,5 +1,7 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
+from builtins import str
+
 from ..exceptions import MalformedFamilies, MalformedValues
 from ..filters import _to_filter
 from ..pb.Client_pb2 import Column, GetRequest, MutateRequest, MutationProto, ScanRequest
@@ -18,6 +20,8 @@ def __init__(self, type, pb):  # noqa: B002
 
 
 def master_request(meta_key):
+    if isinstance(meta_key, str):
+        meta_key = meta_key.encode('utf8')
     rq = GetRequest()
     rq.get.row = meta_key
     rq.get.column.extend(families_to_columns(metaInfoFamily))