diff --git a/salt/cloud/clouds/ec2.py b/salt/cloud/clouds/ec2.py index a197519c42b2..2cffd0b81d31 100644 --- a/salt/cloud/clouds/ec2.py +++ b/salt/cloud/clouds/ec2.py @@ -82,7 +82,6 @@ import binascii import datetime import base64 -import msgpack import re import decimal @@ -91,6 +90,7 @@ import salt.utils.files import salt.utils.hashutils import salt.utils.json +import salt.utils.msgpack import salt.utils.stringutils import salt.utils.yaml from salt._compat import ElementTree as ET @@ -4859,7 +4859,7 @@ def _parse_pricing(url, name): __opts__['cachedir'], 'ec2-pricing-{0}.p'.format(name) ) with salt.utils.files.fopen(outfile, 'w') as fho: - msgpack.dump(regions, fho) + salt.utils.msgpack.dump(regions, fho) return True @@ -4927,7 +4927,8 @@ def show_pricing(kwargs=None, call=None): update_pricing({'type': name}, 'function') with salt.utils.files.fopen(pricefile, 'r') as fhi: - ec2_price = salt.utils.stringutils.to_unicode(msgpack.load(fhi)) + ec2_price = salt.utils.stringutils.to_unicode( + salt.utils.msgpack.load(fhi)) region = get_location(profile) size = profile.get('size', None) diff --git a/salt/cloud/clouds/gce.py b/salt/cloud/clouds/gce.py index dbd51e6c1fa7..d29d2cea99c5 100644 --- a/salt/cloud/clouds/gce.py +++ b/salt/cloud/clouds/gce.py @@ -53,7 +53,6 @@ import re import pprint import logging -import msgpack from ast import literal_eval from salt.utils.versions import LooseVersion as _LooseVersion @@ -91,6 +90,7 @@ import salt.utils.cloud import salt.utils.files import salt.utils.http +import salt.utils.msgpack import salt.config as config from salt.cloud.libcloudfuncs import * # pylint: disable=redefined-builtin,wildcard-import,unused-wildcard-import from salt.exceptions import ( @@ -2628,7 +2628,7 @@ def update_pricing(kwargs=None, call=None): __opts__['cachedir'], 'gce-pricing.p' ) with salt.utils.files.fopen(outfile, 'w') as fho: - msgpack.dump(price_json['dict'], fho) + salt.utils.msgpack.dump(price_json['dict'], fho) return True @@ -2667,7 +2667,7 @@ def show_pricing(kwargs=None, call=None): update_pricing() with salt.utils.files.fopen(pricefile, 'r') as fho: - sizes = msgpack.load(fho) + sizes = salt.utils.msgpack.load(fho) per_hour = float(sizes['gcp_price_list'][size][region]) diff --git a/salt/engines/stalekey.py b/salt/engines/stalekey.py index f927a25049bf..7b27d1733de1 100644 --- a/salt/engines/stalekey.py +++ b/salt/engines/stalekey.py @@ -28,11 +28,11 @@ import salt.key import salt.utils.files import salt.utils.minions +import salt.utils.msgpack import salt.wheel # Import 3rd-party libs from salt.ext import six -import msgpack log = logging.getLogger(__name__) @@ -60,7 +60,7 @@ def start(interval=3600, expire=604800): if os.path.exists(presence_file): try: with salt.utils.files.fopen(presence_file, 'r') as f: - minions = msgpack.load(f) + minions = salt.utils.msgpack.load(f) except IOError as e: log.error('Could not open presence file %s: %s', presence_file, e) time.sleep(interval) @@ -95,7 +95,7 @@ def start(interval=3600, expire=604800): try: with salt.utils.files.fopen(presence_file, 'w') as f: - msgpack.dump(minions, f) + salt.utils.msgpack.dump(minions, f) except IOError as e: log.error('Could not write to presence file %s: %s', presence_file, e) time.sleep(interval) diff --git a/salt/key.py b/salt/key.py index 03d3d7257685..06f64cf62795 100644 --- a/salt/key.py +++ b/salt/key.py @@ -38,9 +38,10 @@ from salt.ext.six.moves import input, zip_longest # pylint: enable=import-error,no-name-in-module,redefined-builtin -# Import third party libs +# We do not always need msgpack, so we do not want to fail here if msgpack is +# not available. try: - import msgpack + import salt.utils.msgpack except ImportError: pass @@ -1044,7 +1045,7 @@ def check_minion_cache(self, preserve_minions=False): if ext == '.json': data = salt.utils.json.load(fp_) elif ext == '.msgpack': - data = msgpack.load(fp_) + data = salt.utils.msgpack.load(fp_) role = salt.utils.stringutils.to_unicode(data['role']) if role not in minions: os.remove(path) diff --git a/salt/log/handlers/fluent_mod.py b/salt/log/handlers/fluent_mod.py index e09f23bbc53f..6508c1231dfe 100644 --- a/salt/log/handlers/fluent_mod.py +++ b/salt/log/handlers/fluent_mod.py @@ -96,14 +96,17 @@ try: # Attempt to import msgpack import msgpack + import salt.utils.msgpack # There is a serialization issue on ARM and potentially other platforms # for some msgpack bindings, check for it if msgpack.loads(msgpack.dumps([1, 2, 3]), use_list=True) is None: raise ImportError + import salt.utils.msgpack except ImportError: # Fall back to msgpack_pure try: import msgpack_pure as msgpack + import salt.utils.msgpack except ImportError: # TODO: Come up with a sane way to get a configured logfile # and write to the logfile when this error is hit also @@ -455,7 +458,7 @@ def _make_packet(self, label, timestamp, data): packet = (tag, timestamp, data) if self.verbose: print(packet) - return msgpack.packb(packet) + return salt.utils.msgpack.packb(packet, _msgpack_module=msgpack) def _send(self, bytes_): self.lock.acquire() diff --git a/salt/modules/saltcheck.py b/salt/modules/saltcheck.py index e4801d6a4e91..e5c02f1ef623 100644 --- a/salt/modules/saltcheck.py +++ b/salt/modules/saltcheck.py @@ -49,7 +49,7 @@ import logging import os import time -from json import loads, dumps +from salt.utils.json import loads, dumps # Import Salt libs import salt.utils.files diff --git a/salt/modules/state.py b/salt/modules/state.py index 0bfd2f943290..dd04300f80f9 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -34,6 +34,7 @@ import salt.utils.hashutils import salt.utils.jid import salt.utils.json +import salt.utils.msgpack import salt.utils.platform import salt.utils.state import salt.utils.stringutils @@ -44,7 +45,6 @@ # Import 3rd-party libs from salt.ext import six -import msgpack __proxyenabled__ = ['*'] @@ -184,7 +184,7 @@ def _get_pause(jid, state_id=None): data[state_id] = {} if os.path.exists(pause_path): with salt.utils.files.fopen(pause_path, 'rb') as fp_: - data = msgpack.loads(fp_.read()) + data = salt.utils.msgpack.loads(fp_.read()) return data, pause_path @@ -255,7 +255,7 @@ def soft_kill(jid, state_id=None): data, pause_path = _get_pause(jid, state_id) data[state_id]['kill'] = True with salt.utils.files.fopen(pause_path, 'wb') as fp_: - fp_.write(msgpack.dumps(data)) + fp_.write(salt.utils.msgpack.dumps(data)) def pause(jid, state_id=None, duration=None): @@ -290,7 +290,7 @@ def pause(jid, state_id=None, duration=None): if duration: data[state_id]['duration'] = int(duration) with salt.utils.files.fopen(pause_path, 'wb') as fp_: - fp_.write(msgpack.dumps(data)) + fp_.write(salt.utils.msgpack.dumps(data)) def resume(jid, state_id=None): @@ -324,7 +324,7 @@ def resume(jid, state_id=None): if state_id == '__all__': data = {} with salt.utils.files.fopen(pause_path, 'wb') as fp_: - fp_.write(msgpack.dumps(data)) + fp_.write(salt.utils.msgpack.dumps(data)) def orchestrate(mods, diff --git a/salt/modules/win_repo.py b/salt/modules/win_repo.py index c0a9dde4536d..21f35164df07 100644 --- a/salt/modules/win_repo.py +++ b/salt/modules/win_repo.py @@ -31,11 +31,8 @@ PER_REMOTE_ONLY ) from salt.ext import six -try: - import msgpack -except ImportError: - import msgpack_pure as msgpack # pylint: disable=import-error import salt.utils.gitfs +import salt.utils.msgpack # pylint: enable=unused-import log = logging.getLogger(__name__) diff --git a/salt/payload.py b/salt/payload.py index 5df6458b861a..aa6b8f8792d9 100644 --- a/salt/payload.py +++ b/salt/payload.py @@ -55,6 +55,10 @@ #sys.exit(salt.defaults.exitcodes.EX_GENERIC) +if HAS_MSGPACK: + import salt.utils.msgpack + + if HAS_MSGPACK and not hasattr(msgpack, 'exceptions'): class PackValueError(Exception): ''' @@ -75,14 +79,15 @@ def package(payload): This method for now just wraps msgpack.dumps, but it is here so that we can make the serialization a custom option in the future with ease. ''' - return msgpack.dumps(payload) + return salt.utils.msgpack.dumps(payload, _msgpack_module=msgpack) def unpackage(package_): ''' Unpackages a payload ''' - return msgpack.loads(package_, use_list=True) + return salt.utils.msgpack.loads(package_, use_list=True, + _msgpack_module=msgpack) def format_payload(enc, **kwargs): @@ -142,12 +147,17 @@ def ext_type_decoder(code, data): # that under Python 2 we can still work with older versions # of msgpack. try: - ret = msgpack.loads(msg, use_list=True, ext_hook=ext_type_decoder, encoding=encoding) + ret = salt.utils.msgpack.loads(msg, use_list=True, + ext_hook=ext_type_decoder, + encoding=encoding, + _msgpack_module=msgpack) except UnicodeDecodeError: # msg contains binary data ret = msgpack.loads(msg, use_list=True, ext_hook=ext_type_decoder) else: - ret = msgpack.loads(msg, use_list=True, ext_hook=ext_type_decoder) + ret = salt.utils.msgpack.loads(msg, use_list=True, + ext_hook=ext_type_decoder, + _msgpack_module=msgpack) if six.PY3 and encoding is None and not raw: ret = salt.transport.frame.decode_embedded_strs(ret) except Exception as exc: @@ -214,9 +224,12 @@ def ext_type_encoder(obj): # Due to this, if we don't need it, don't pass it at all so # that under Python 2 we can still work with older versions # of msgpack. - return msgpack.dumps(msg, default=ext_type_encoder, use_bin_type=use_bin_type) + return salt.utils.msgpack.dumps(msg, default=ext_type_encoder, + use_bin_type=use_bin_type, + _msgpack_module=msgpack) else: - return msgpack.dumps(msg, default=ext_type_encoder) + return salt.utils.msgpack.dumps(msg, default=ext_type_encoder, + _msgpack_module=msgpack) except (OverflowError, msgpack.exceptions.PackValueError): # msgpack<=0.4.6 don't call ext encoder on very long integers raising the error instead. # Convert any very long longs to strings and call dumps again. @@ -239,9 +252,12 @@ def verylong_encoder(obj): msg = verylong_encoder(msg) if msgpack.version >= (0, 4, 0): - return msgpack.dumps(msg, default=ext_type_encoder, use_bin_type=use_bin_type) + return salt.utils.msgpack.dumps(msg, default=ext_type_encoder, + use_bin_type=use_bin_type, + _msgpack_module=msgpack) else: - return msgpack.dumps(msg, default=ext_type_encoder) + return salt.utils.msgpack.dumps(msg, default=ext_type_encoder, + _msgpack_module=msgpack) def dump(self, msg, fn_): ''' diff --git a/salt/renderers/msgpack.py b/salt/renderers/msgpack.py index f58d11b85b8d..eceac4f53bb5 100644 --- a/salt/renderers/msgpack.py +++ b/salt/renderers/msgpack.py @@ -1,10 +1,8 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals -# Import third party libs -import msgpack - # Import salt libs +import salt.utils.msgpack from salt.ext import six @@ -28,4 +26,4 @@ def render(msgpack_data, saltenv='base', sls='', **kws): msgpack_data = msgpack_data[(msgpack_data.find('\n') + 1):] if not msgpack_data.strip(): return {} - return msgpack.loads(msgpack_data) + return salt.utils.msgpack.loads(msgpack_data) diff --git a/salt/returners/local_cache.py b/salt/returners/local_cache.py index 6799730877be..721bd70045e6 100644 --- a/salt/returners/local_cache.py +++ b/salt/returners/local_cache.py @@ -20,11 +20,11 @@ import salt.utils.files import salt.utils.jid import salt.utils.minions +import salt.utils.msgpack import salt.utils.stringutils import salt.exceptions # Import 3rd-party libs -import msgpack from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin @@ -514,7 +514,7 @@ def save_reg(data): raise try: with salt.utils.files.fopen(regfile, 'a') as fh_: - msgpack.dump(data, fh_) + salt.utils.msgpack.dump(data, fh_) except: log.error('Could not write to msgpack file %s', __opts__['outdir']) raise @@ -528,7 +528,7 @@ def load_reg(): regfile = os.path.join(reg_dir, 'register') try: with salt.utils.files.fopen(regfile, 'r') as fh_: - return msgpack.load(fh_) + return salt.utils.msgpack.load(fh_) except: log.error('Could not write to msgpack file %s', __opts__['outdir']) raise diff --git a/salt/runners/winrepo.py b/salt/runners/winrepo.py index 480a3138b66b..321b91638eb8 100644 --- a/salt/runners/winrepo.py +++ b/salt/runners/winrepo.py @@ -12,15 +12,12 @@ # Import third party libs from salt.ext import six -try: - import msgpack -except ImportError: - import msgpack_pure as msgpack # pylint: disable=import-error # Import salt libs from salt.exceptions import CommandExecutionError, SaltRenderError import salt.utils.files import salt.utils.gitfs +import salt.utils.msgpack import salt.utils.path import logging import salt.minion @@ -124,7 +121,7 @@ def genrepo(opts=None, fire_event=True): ret.setdefault('name_map', {}).update(revmap) with salt.utils.files.fopen( os.path.join(winrepo_dir, winrepo_cachefile), 'w+b') as repo: - repo.write(msgpack.dumps(ret)) + repo.write(salt.utils.msgpack.dumps(ret)) return ret diff --git a/salt/sdb/sqlite3.py b/salt/sdb/sqlite3.py index 540a289d56ae..006d574c283b 100644 --- a/salt/sdb/sqlite3.py +++ b/salt/sdb/sqlite3.py @@ -54,11 +54,9 @@ HAS_SQLITE3 = False # Import salt libs +import salt.utils.msgpack from salt.ext import six -# Import third party libs -import msgpack - DEFAULT_TABLE = 'sdb' @@ -126,9 +124,9 @@ def set_(key, value, profile=None): return False conn, cur, table = _connect(profile) if six.PY2: - value = buffer(msgpack.packb(value)) + value = buffer(salt.utils.msgpack.packb(value)) else: - value = memoryview(msgpack.packb(value)) + value = memoryview(salt.utils.msgpack.packb(value)) q = profile.get('set_query', ('INSERT OR REPLACE INTO {0} VALUES ' '(:key, :value)').format(table)) conn.execute(q, {'key': key, 'value': value}) @@ -149,4 +147,4 @@ def get(key, profile=None): res = res.fetchone() if not res: return None - return msgpack.unpackb(res[0]) + return salt.utils.msgpack.unpackb(res[0]) diff --git a/salt/serializers/msgpack.py b/salt/serializers/msgpack.py index f55fa878b669..90df192ee3c6 100644 --- a/salt/serializers/msgpack.py +++ b/salt/serializers/msgpack.py @@ -24,6 +24,7 @@ try: # Attempt to import msgpack import msgpack + import salt.utils.msgpack # There is a serialization issue on ARM and potentially other platforms # for some msgpack bindings, check for it if msgpack.loads(msgpack.dumps([1, 2, 3]), use_list=True) is None: @@ -33,6 +34,7 @@ # Fall back to msgpack_pure try: import msgpack_pure as msgpack # pylint: disable=import-error + import salt.utils.msgpack except ImportError: # TODO: Come up with a sane way to get a configured logfile # and write to the logfile when this error is hit also @@ -60,7 +62,8 @@ def _deserialize(stream_or_string, **options): def _serialize(obj, **options): try: - return msgpack.dumps(obj, **options) + return salt.utils.msgpack.dumps(obj, _msgpack_module=msgpack, + **options) except Exception as error: raise SerializationError(error) @@ -68,7 +71,9 @@ def _deserialize(stream_or_string, **options): try: options.setdefault('use_list', True) options.setdefault('encoding', 'utf-8') - return msgpack.loads(stream_or_string, **options) + return salt.utils.msgpack.loads(stream_or_string, + _msgpack_module=msgpack, + **options) except Exception as error: raise DeserializationError(error) @@ -95,14 +100,16 @@ def _decoder(obj): def _serialize(obj, **options): try: obj = _encoder(obj) - return msgpack.dumps(obj, **options) + return salt.utils.msgpack.dumps(obj, _msgpack_module=msgpack, + **options) except Exception as error: raise SerializationError(error) def _deserialize(stream_or_string, **options): options.setdefault('use_list', True) try: - obj = msgpack.loads(stream_or_string) + obj = salt.utils.msgpack.loads(stream_or_string, + _msgpack_module=msgpack) return _decoder(obj) except Exception as error: raise DeserializationError(error) diff --git a/salt/state.py b/salt/state.py index 80f40f5e6711..be1821093311 100644 --- a/salt/state.py +++ b/salt/state.py @@ -38,6 +38,7 @@ import salt.utils.event import salt.utils.files import salt.utils.immutabletypes as immutabletypes +import salt.utils.msgpack as msgpack import salt.utils.platform import salt.utils.process import salt.utils.url @@ -54,7 +55,6 @@ import salt.utils.yamlloader as yamlloader # Import third party libs -import msgpack # pylint: disable=import-error,no-name-in-module,redefined-builtin from salt.ext import six from salt.ext.six.moves import map, range, reload_module diff --git a/salt/states/netsnmp.py b/salt/states/netsnmp.py index 9bafffefbc93..7fcdf6fc4132 100644 --- a/salt/states/netsnmp.py +++ b/salt/states/netsnmp.py @@ -23,9 +23,8 @@ import logging log = logging.getLogger(__name__) -from json import loads, dumps - # salt lib +from salt.utils.json import loads, dumps from salt.ext import six # import NAPALM utils import salt.utils.napalm diff --git a/salt/states/netusers.py b/salt/states/netusers.py index 58b3751c6b74..f4ff648fbfdd 100644 --- a/salt/states/netusers.py +++ b/salt/states/netusers.py @@ -25,9 +25,9 @@ # Python std lib from copy import deepcopy -from json import loads, dumps # salt lib +from salt.utils.json import loads, dumps from salt.ext import six # import NAPALM utils import salt.utils.napalm diff --git a/salt/states/pkg.py b/salt/states/pkg.py index eed114d9cec4..07178871aab6 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -135,10 +135,6 @@ # The following imports are used by the namespaced win_pkg funcs # and need to be included in their globals. # pylint: disable=import-error,unused-import - try: - import msgpack - except ImportError: - import msgpack_pure as msgpack from salt.utils.versions import LooseVersion # pylint: enable=import-error,unused-import # pylint: enable=invalid-name diff --git a/salt/states/probes.py b/salt/states/probes.py index e621fb353606..f79eeb71d027 100644 --- a/salt/states/probes.py +++ b/salt/states/probes.py @@ -25,9 +25,9 @@ log = logging.getLogger(__name__) from copy import deepcopy -from json import loads, dumps # salt modules +from salt.utils.json import loads, dumps from salt.ext import six # import NAPALM utils import salt.utils.napalm diff --git a/salt/states/zabbix_host.py b/salt/states/zabbix_host.py index 85f0b322efda..85e173bf2f71 100644 --- a/salt/states/zabbix_host.py +++ b/salt/states/zabbix_host.py @@ -7,7 +7,7 @@ ''' from __future__ import absolute_import, print_function, unicode_literals -from json import loads, dumps +from salt.utils.json import loads, dumps from copy import deepcopy from salt.ext import six diff --git a/salt/states/zabbix_user.py b/salt/states/zabbix_user.py index 519d236290ad..e6dd2eb90c2e 100644 --- a/salt/states/zabbix_user.py +++ b/salt/states/zabbix_user.py @@ -9,7 +9,7 @@ # Import Python libs from __future__ import absolute_import, print_function, unicode_literals -from json import loads, dumps +from salt.utils.json import loads, dumps from copy import deepcopy # Import Salt libs diff --git a/salt/transport/frame.py b/salt/transport/frame.py index 33d0c0d91703..88b595184ec7 100644 --- a/salt/transport/frame.py +++ b/salt/transport/frame.py @@ -4,7 +4,7 @@ ''' # Import python libs from __future__ import absolute_import, print_function, unicode_literals -import msgpack +import salt.utils.msgpack from salt.ext import six @@ -18,7 +18,7 @@ def frame_msg(body, header=None, raw_body=False): # pylint: disable=unused-argu framed_msg['head'] = header framed_msg['body'] = body - return msgpack.dumps(framed_msg) + return salt.utils.msgpack.dumps(framed_msg) def frame_msg_ipc(body, header=None, raw_body=False): # pylint: disable=unused-argument @@ -35,9 +35,9 @@ def frame_msg_ipc(body, header=None, raw_body=False): # pylint: disable=unused- framed_msg['head'] = header framed_msg['body'] = body if six.PY2: - return msgpack.dumps(framed_msg) + return salt.utils.msgpack.dumps(framed_msg) else: - return msgpack.dumps(framed_msg, use_bin_type=True) + return salt.utils.msgpack.dumps(framed_msg, use_bin_type=True) def _decode_embedded_list(src): diff --git a/salt/transport/ipc.py b/salt/transport/ipc.py index f1f844bca02b..84f87928b71b 100644 --- a/salt/transport/ipc.py +++ b/salt/transport/ipc.py @@ -10,9 +10,6 @@ import weakref import time -# Import 3rd-party libs -import msgpack - # Import Tornado libs import tornado import tornado.gen @@ -24,6 +21,7 @@ # Import Salt libs import salt.transport.client import salt.transport.frame +import salt.utils.msgpack as msgpack from salt.ext import six log = logging.getLogger(__name__) diff --git a/salt/transport/tcp.py b/salt/transport/tcp.py index 598ea07a8b5d..9450ee0ad93e 100644 --- a/salt/transport/tcp.py +++ b/salt/transport/tcp.py @@ -10,7 +10,6 @@ from __future__ import absolute_import, print_function, unicode_literals import errno import logging -import msgpack import socket import os import weakref @@ -22,6 +21,7 @@ import salt.utils.asynchronous import salt.utils.event import salt.utils.files +import salt.utils.msgpack as msgpack import salt.utils.platform import salt.utils.process import salt.utils.verify diff --git a/salt/utils/cache.py b/salt/utils/cache.py index 6581af9f870d..adf34fb8f074 100644 --- a/salt/utils/cache.py +++ b/salt/utils/cache.py @@ -9,7 +9,7 @@ import time import logging try: - import msgpack + import salt.utils.msgpack HAS_MSGPACK = True except ImportError: HAS_MSGPACK = False @@ -139,7 +139,9 @@ def _read(self): if not HAS_MSGPACK or not os.path.exists(self._path): return with salt.utils.files.fopen(self._path, 'rb') as fp_: - cache = salt.utils.data.decode(msgpack.load(fp_, encoding=__salt_system_encoding__)) + cache = salt.utils.data.decode( + salt.utils.msgpack.load( + fp_, encoding=__salt_system_encoding__)) if "CacheDisk_cachetime" in cache: # new format self._dict = cache["CacheDisk_data"] self._key_cache_time = cache["CacheDisk_cachetime"] @@ -164,7 +166,7 @@ def _write(self): "CacheDisk_data": self._dict, "CacheDisk_cachetime": self._key_cache_time } - msgpack.dump(cache, fp_, use_bin_type=True) + salt.utils.msgpack.dump(cache, fp_, use_bin_type=True) class CacheCli(object): diff --git a/salt/utils/cloud.py b/salt/utils/cloud.py index c1ee50afa509..64dff483f5e1 100644 --- a/salt/utils/cloud.py +++ b/salt/utils/cloud.py @@ -18,7 +18,6 @@ import multiprocessing import logging import pipes -import msgpack import traceback import copy import re @@ -51,6 +50,7 @@ import salt.utils.data import salt.utils.event import salt.utils.files +import salt.utils.msgpack import salt.utils.platform import salt.utils.stringutils import salt.utils.versions @@ -2510,7 +2510,9 @@ def cachedir_index_add(minion_id, profile, driver, provider, base=None): if os.path.exists(index_file): mode = 'rb' if six.PY3 else 'r' with salt.utils.files.fopen(index_file, mode) as fh_: - index = salt.utils.data.decode(msgpack.load(fh_, encoding=MSGPACK_ENCODING)) + index = salt.utils.data.decode( + salt.utils.msgpack.msgpack.load( + fh_, encoding=MSGPACK_ENCODING)) else: index = {} @@ -2527,7 +2529,7 @@ def cachedir_index_add(minion_id, profile, driver, provider, base=None): mode = 'wb' if six.PY3 else 'w' with salt.utils.files.fopen(index_file, mode) as fh_: - msgpack.dump(index, fh_, encoding=MSGPACK_ENCODING) + salt.utils.msgpack.dump(index, fh_, encoding=MSGPACK_ENCODING) unlock_file(index_file) @@ -2544,7 +2546,8 @@ def cachedir_index_del(minion_id, base=None): if os.path.exists(index_file): mode = 'rb' if six.PY3 else 'r' with salt.utils.files.fopen(index_file, mode) as fh_: - index = salt.utils.data.decode(msgpack.load(fh_, encoding=MSGPACK_ENCODING)) + index = salt.utils.data.decode( + salt.utils.msgpack.load(fh_, encoding=MSGPACK_ENCODING)) else: return @@ -2553,7 +2556,7 @@ def cachedir_index_del(minion_id, base=None): mode = 'wb' if six.PY3 else 'w' with salt.utils.files.fopen(index_file, mode) as fh_: - msgpack.dump(index, fh_, encoding=MSGPACK_ENCODING) + salt.utils.msgpack.dump(index, fh_, encoding=MSGPACK_ENCODING) unlock_file(index_file) @@ -2611,7 +2614,7 @@ def request_minion_cachedir( path = os.path.join(base, 'requested', fname) mode = 'wb' if six.PY3 else 'w' with salt.utils.files.fopen(path, mode) as fh_: - msgpack.dump(data, fh_, encoding=MSGPACK_ENCODING) + salt.utils.msgpack.dump(data, fh_, encoding=MSGPACK_ENCODING) def change_minion_cachedir( @@ -2643,12 +2646,13 @@ def change_minion_cachedir( path = os.path.join(base, cachedir, fname) with salt.utils.files.fopen(path, 'r') as fh_: - cache_data = salt.utils.data.decode(msgpack.load(fh_, encoding=MSGPACK_ENCODING)) + cache_data = salt.utils.data.decode( + salt.utils.msgpack.load(fh_, encoding=MSGPACK_ENCODING)) cache_data.update(data) with salt.utils.files.fopen(path, 'w') as fh_: - msgpack.dump(cache_data, fh_, encoding=MSGPACK_ENCODING) + salt.utils.msgpack.dump(cache_data, fh_, encoding=MSGPACK_ENCODING) def activate_minion_cachedir(minion_id, base=None): @@ -2722,7 +2726,8 @@ def list_cache_nodes_full(opts=None, provider=None, base=None): minion_id = fname[:-2] # strip '.p' from end of msgpack filename mode = 'rb' if six.PY3 else 'r' with salt.utils.files.fopen(fpath, mode) as fh_: - minions[driver][prov][minion_id] = salt.utils.data.decode(msgpack.load(fh_, encoding=MSGPACK_ENCODING)) + minions[driver][prov][minion_id] = salt.utils.data.decode( + salt.utils.msgpack.load(fh_, encoding=MSGPACK_ENCODING)) return minions @@ -2899,7 +2904,7 @@ def cache_node_list(nodes, provider, opts): path = os.path.join(prov_dir, '{0}.p'.format(node)) mode = 'wb' if six.PY3 else 'w' with salt.utils.files.fopen(path, mode) as fh_: - msgpack.dump(nodes[node], fh_, encoding=MSGPACK_ENCODING) + salt.utils.msgpack.dump(nodes[node], fh_, encoding=MSGPACK_ENCODING) def cache_node(node, provider, opts): @@ -2925,7 +2930,7 @@ def cache_node(node, provider, opts): path = os.path.join(prov_dir, '{0}.p'.format(node['name'])) mode = 'wb' if six.PY3 else 'w' with salt.utils.files.fopen(path, mode) as fh_: - msgpack.dump(node, fh_, encoding=MSGPACK_ENCODING) + salt.utils.msgpack.dump(node, fh_, encoding=MSGPACK_ENCODING) def missing_node_cache(prov_dir, node_list, provider, opts): @@ -3000,7 +3005,8 @@ def diff_node_cache(prov_dir, node, new_data, opts): with salt.utils.files.fopen(path, 'r') as fh_: try: - cache_data = salt.utils.data.decode(msgpack.load(fh_, encoding=MSGPACK_ENCODING)) + cache_data = salt.utils.data.decode( + salt.utils.msgpack.load(fh_, encoding=MSGPACK_ENCODING)) except ValueError: log.warning('Cache for %s was corrupt: Deleting', node) cache_data = {} diff --git a/salt/utils/http.py b/salt/utils/http.py index ee511d37fed4..97d25db9ba99 100644 --- a/salt/utils/http.py +++ b/salt/utils/http.py @@ -83,7 +83,7 @@ HAS_REQUESTS = False try: - import msgpack + import salt.utils.msgpack HAS_MSGPACK = True except ImportError: HAS_MSGPACK = False @@ -274,12 +274,12 @@ def query(url, # contain expirations, they can't be stored in a proper cookie jar. if os.path.isfile(session_cookie_jar): with salt.utils.files.fopen(session_cookie_jar, 'rb') as fh_: - session_cookies = msgpack.load(fh_) + session_cookies = salt.utils.msgpack.load(fh_) if isinstance(session_cookies, dict): header_dict.update(session_cookies) else: with salt.utils.files.fopen(session_cookie_jar, 'wb') as fh_: - msgpack.dump('', fh_) + salt.utils.msgpack.dump('', fh_) for header in header_list: comps = header.split(':') @@ -630,9 +630,9 @@ def query(url, with salt.utils.files.fopen(session_cookie_jar, 'wb') as fh_: session_cookies = result_headers.get('set-cookie', None) if session_cookies is not None: - msgpack.dump({'Cookie': session_cookies}, fh_) + salt.utils.msgpack.dump({'Cookie': session_cookies}, fh_) else: - msgpack.dump('', fh_) + salt.utils.msgpack.dump('', fh_) if status is True: ret['status'] = result_status_code diff --git a/salt/utils/msgpack.py b/salt/utils/msgpack.py new file mode 100644 index 000000000000..7e66cb8ed739 --- /dev/null +++ b/salt/utils/msgpack.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +''' +Functions to work with MessagePack +''' + +from __future__ import absolute_import + +# Import Python libs +try: + # Attempt to import msgpack + import msgpack +except ImportError: + # Fall back to msgpack_pure + import msgpack_pure as msgpack # pylint: disable=import-error + + +def pack(o, stream, **kwargs): + ''' + .. versionadded:: 2018.3.4 + + Wraps msgpack.pack and ensures that the passed object is unwrapped if it is + a proxy. + + By default, this function uses the msgpack module and falls back to + msgpack_pure, if the msgpack is not available. You can pass an alternate + msgpack module using the _msgpack_module argument. + ''' + msgpack_module = kwargs.pop('_msgpack_module', msgpack) + return msgpack_module.pack(o, stream, **kwargs) + + +def packb(o, **kwargs): + ''' + .. versionadded:: 2018.3.4 + + Wraps msgpack.packb and ensures that the passed object is unwrapped if it + is a proxy. + + By default, this function uses the msgpack module and falls back to + msgpack_pure, if the msgpack is not available. You can pass an alternate + msgpack module using the _msgpack_module argument. + ''' + msgpack_module = kwargs.pop('_msgpack_module', msgpack) + return msgpack_module.packb(o, **kwargs) + + +def unpack(stream, **kwargs): + ''' + .. versionadded:: 2018.3.4 + + Wraps msgpack.unpack. + + By default, this function uses the msgpack module and falls back to + msgpack_pure, if the msgpack is not available. You can pass an alternate + msgpack module using the _msgpack_module argument. + ''' + msgpack_module = kwargs.pop('_msgpack_module', msgpack) + return msgpack_module.unpack(stream, **kwargs) + + +def unpackb(packed, **kwargs): + ''' + .. versionadded:: 2018.3.4 + + Wraps msgpack.unpack. + + By default, this function uses the msgpack module and falls back to + msgpack_pure, if the msgpack is not available. You can pass an alternate + msgpack module using the _msgpack_module argument. + ''' + msgpack_module = kwargs.pop('_msgpack_module', msgpack) + return msgpack_module.unpackb(packed, **kwargs) + + +# alias for compatibility to simplejson/marshal/pickle. +load = unpack +loads = unpackb + +dump = pack +dumps = packb diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index e706802aa74c..3373f87602cb 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -50,6 +50,7 @@ import salt.version import salt.utils.color import salt.utils.files +import salt.utils.msgpack as msgpack import salt.utils.path import salt.utils.platform import salt.utils.process @@ -68,7 +69,6 @@ pass # Import 3rd-party libs -import msgpack from salt.ext import six from salt.ext.six.moves import cStringIO diff --git a/tests/integration/files/log_handlers/runtests_log_handler.py b/tests/integration/files/log_handlers/runtests_log_handler.py index 983447d97ae6..2269c063ff83 100644 --- a/tests/integration/files/log_handlers/runtests_log_handler.py +++ b/tests/integration/files/log_handlers/runtests_log_handler.py @@ -19,10 +19,8 @@ import threading from multiprocessing import Queue -# Import 3rd-party libs -import msgpack - # Import Salt libs +import salt.utils.msgpack from salt.ext import six import salt.log.setup @@ -85,7 +83,8 @@ def process_queue(port, queue): break # Just log everything, filtering will happen on the main process # logging handlers - sock.sendall(msgpack.dumps(record.__dict__, encoding='utf-8')) + sock.sendall(salt.utils.msgpack.dumps(record.__dict__, + encoding='utf-8')) except (IOError, EOFError, KeyboardInterrupt, SystemExit): sock.shutdown(socket.SHUT_RDWR) sock.close() diff --git a/tests/packdump.py b/tests/packdump.py index 92ed79de29bc..5a230eed946f 100644 --- a/tests/packdump.py +++ b/tests/packdump.py @@ -9,8 +9,8 @@ import sys import pprint -# Import third party libs -import msgpack +# Import Salt libs +import salt.utils.msgpack def dump(path): @@ -21,7 +21,7 @@ def dump(path): print('Not a file') return with open(path, 'rb') as fp_: - data = msgpack.loads(fp_.read()) + data = salt.utils.msgpack.loads(fp_.read()) pprint.pprint(data)