From 013335340fb90a839e0ae207c63decf27f1d0211 Mon Sep 17 00:00:00 2001 From: Sebastian Marsching Date: Tue, 27 Nov 2018 20:04:31 +0100 Subject: [PATCH] Consistently use salt.utils.json and salt.utils.msgpack. This commit introduces salt.utils.msgpack modifies all places in the code that either use json or msgpack to use salt.utils.json or salt.utils.msgpack respectively. While this change itself does not have any effect, it is important to allow for centrally dealing with objects that cannot be directly serialied via json or msgpack. --- salt/cloud/clouds/ec2.py | 7 +- salt/cloud/clouds/gce.py | 6 +- salt/engines/stalekey.py | 6 +- salt/key.py | 7 +- salt/log/handlers/fluent_mod.py | 5 +- salt/modules/saltcheck.py | 2 +- salt/modules/state.py | 10 +-- salt/modules/win_repo.py | 5 +- salt/payload.py | 32 ++++++-- salt/renderers/msgpack.py | 6 +- salt/returners/local_cache.py | 6 +- salt/runners/winrepo.py | 7 +- salt/sdb/sqlite3.py | 10 +-- salt/serializers/msgpack.py | 15 +++- salt/state.py | 2 +- salt/states/netsnmp.py | 3 +- salt/states/netusers.py | 2 +- salt/states/pkg.py | 4 - salt/states/probes.py | 2 +- salt/states/zabbix_host.py | 2 +- salt/states/zabbix_user.py | 2 +- salt/transport/frame.py | 8 +- salt/transport/ipc.py | 4 +- salt/transport/tcp.py | 2 +- salt/utils/cache.py | 8 +- salt/utils/cloud.py | 30 ++++--- salt/utils/http.py | 10 +-- salt/utils/msgpack.py | 80 +++++++++++++++++++ tests/integration/__init__.py | 2 +- .../log_handlers/runtests_log_handler.py | 7 +- tests/packdump.py | 6 +- 31 files changed, 198 insertions(+), 100 deletions(-) create mode 100644 salt/utils/msgpack.py diff --git a/salt/cloud/clouds/ec2.py b/salt/cloud/clouds/ec2.py index a197519c42b2..2cffd0b81d31 100644 --- a/salt/cloud/clouds/ec2.py +++ b/salt/cloud/clouds/ec2.py @@ -82,7 +82,6 @@ import binascii import datetime import base64 -import msgpack import re import decimal @@ -91,6 +90,7 @@ import salt.utils.files import salt.utils.hashutils import salt.utils.json +import salt.utils.msgpack import salt.utils.stringutils import salt.utils.yaml from salt._compat import ElementTree as ET @@ -4859,7 +4859,7 @@ def _parse_pricing(url, name): __opts__['cachedir'], 'ec2-pricing-{0}.p'.format(name) ) with salt.utils.files.fopen(outfile, 'w') as fho: - msgpack.dump(regions, fho) + salt.utils.msgpack.dump(regions, fho) return True @@ -4927,7 +4927,8 @@ def show_pricing(kwargs=None, call=None): update_pricing({'type': name}, 'function') with salt.utils.files.fopen(pricefile, 'r') as fhi: - ec2_price = salt.utils.stringutils.to_unicode(msgpack.load(fhi)) + ec2_price = salt.utils.stringutils.to_unicode( + salt.utils.msgpack.load(fhi)) region = get_location(profile) size = profile.get('size', None) diff --git a/salt/cloud/clouds/gce.py b/salt/cloud/clouds/gce.py index dbd51e6c1fa7..d29d2cea99c5 100644 --- a/salt/cloud/clouds/gce.py +++ b/salt/cloud/clouds/gce.py @@ -53,7 +53,6 @@ import re import pprint import logging -import msgpack from ast import literal_eval from salt.utils.versions import LooseVersion as _LooseVersion @@ -91,6 +90,7 @@ import salt.utils.cloud import salt.utils.files import salt.utils.http +import salt.utils.msgpack import salt.config as config from salt.cloud.libcloudfuncs import * # pylint: disable=redefined-builtin,wildcard-import,unused-wildcard-import from salt.exceptions import ( @@ -2628,7 +2628,7 @@ def update_pricing(kwargs=None, call=None): __opts__['cachedir'], 'gce-pricing.p' ) with salt.utils.files.fopen(outfile, 'w') as fho: - msgpack.dump(price_json['dict'], fho) + salt.utils.msgpack.dump(price_json['dict'], fho) return True @@ -2667,7 +2667,7 @@ def show_pricing(kwargs=None, call=None): update_pricing() with salt.utils.files.fopen(pricefile, 'r') as fho: - sizes = msgpack.load(fho) + sizes = salt.utils.msgpack.load(fho) per_hour = float(sizes['gcp_price_list'][size][region]) diff --git a/salt/engines/stalekey.py b/salt/engines/stalekey.py index f927a25049bf..7b27d1733de1 100644 --- a/salt/engines/stalekey.py +++ b/salt/engines/stalekey.py @@ -28,11 +28,11 @@ import salt.key import salt.utils.files import salt.utils.minions +import salt.utils.msgpack import salt.wheel # Import 3rd-party libs from salt.ext import six -import msgpack log = logging.getLogger(__name__) @@ -60,7 +60,7 @@ def start(interval=3600, expire=604800): if os.path.exists(presence_file): try: with salt.utils.files.fopen(presence_file, 'r') as f: - minions = msgpack.load(f) + minions = salt.utils.msgpack.load(f) except IOError as e: log.error('Could not open presence file %s: %s', presence_file, e) time.sleep(interval) @@ -95,7 +95,7 @@ def start(interval=3600, expire=604800): try: with salt.utils.files.fopen(presence_file, 'w') as f: - msgpack.dump(minions, f) + salt.utils.msgpack.dump(minions, f) except IOError as e: log.error('Could not write to presence file %s: %s', presence_file, e) time.sleep(interval) diff --git a/salt/key.py b/salt/key.py index 03d3d7257685..06f64cf62795 100644 --- a/salt/key.py +++ b/salt/key.py @@ -38,9 +38,10 @@ from salt.ext.six.moves import input, zip_longest # pylint: enable=import-error,no-name-in-module,redefined-builtin -# Import third party libs +# We do not always need msgpack, so we do not want to fail here if msgpack is +# not available. try: - import msgpack + import salt.utils.msgpack except ImportError: pass @@ -1044,7 +1045,7 @@ def check_minion_cache(self, preserve_minions=False): if ext == '.json': data = salt.utils.json.load(fp_) elif ext == '.msgpack': - data = msgpack.load(fp_) + data = salt.utils.msgpack.load(fp_) role = salt.utils.stringutils.to_unicode(data['role']) if role not in minions: os.remove(path) diff --git a/salt/log/handlers/fluent_mod.py b/salt/log/handlers/fluent_mod.py index e09f23bbc53f..6508c1231dfe 100644 --- a/salt/log/handlers/fluent_mod.py +++ b/salt/log/handlers/fluent_mod.py @@ -96,14 +96,17 @@ try: # Attempt to import msgpack import msgpack + import salt.utils.msgpack # There is a serialization issue on ARM and potentially other platforms # for some msgpack bindings, check for it if msgpack.loads(msgpack.dumps([1, 2, 3]), use_list=True) is None: raise ImportError + import salt.utils.msgpack except ImportError: # Fall back to msgpack_pure try: import msgpack_pure as msgpack + import salt.utils.msgpack except ImportError: # TODO: Come up with a sane way to get a configured logfile # and write to the logfile when this error is hit also @@ -455,7 +458,7 @@ def _make_packet(self, label, timestamp, data): packet = (tag, timestamp, data) if self.verbose: print(packet) - return msgpack.packb(packet) + return salt.utils.msgpack.packb(packet, _msgpack_module=msgpack) def _send(self, bytes_): self.lock.acquire() diff --git a/salt/modules/saltcheck.py b/salt/modules/saltcheck.py index e4801d6a4e91..e5c02f1ef623 100644 --- a/salt/modules/saltcheck.py +++ b/salt/modules/saltcheck.py @@ -49,7 +49,7 @@ import logging import os import time -from json import loads, dumps +from salt.utils.json import loads, dumps # Import Salt libs import salt.utils.files diff --git a/salt/modules/state.py b/salt/modules/state.py index 0bfd2f943290..dd04300f80f9 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -34,6 +34,7 @@ import salt.utils.hashutils import salt.utils.jid import salt.utils.json +import salt.utils.msgpack import salt.utils.platform import salt.utils.state import salt.utils.stringutils @@ -44,7 +45,6 @@ # Import 3rd-party libs from salt.ext import six -import msgpack __proxyenabled__ = ['*'] @@ -184,7 +184,7 @@ def _get_pause(jid, state_id=None): data[state_id] = {} if os.path.exists(pause_path): with salt.utils.files.fopen(pause_path, 'rb') as fp_: - data = msgpack.loads(fp_.read()) + data = salt.utils.msgpack.loads(fp_.read()) return data, pause_path @@ -255,7 +255,7 @@ def soft_kill(jid, state_id=None): data, pause_path = _get_pause(jid, state_id) data[state_id]['kill'] = True with salt.utils.files.fopen(pause_path, 'wb') as fp_: - fp_.write(msgpack.dumps(data)) + fp_.write(salt.utils.msgpack.dumps(data)) def pause(jid, state_id=None, duration=None): @@ -290,7 +290,7 @@ def pause(jid, state_id=None, duration=None): if duration: data[state_id]['duration'] = int(duration) with salt.utils.files.fopen(pause_path, 'wb') as fp_: - fp_.write(msgpack.dumps(data)) + fp_.write(salt.utils.msgpack.dumps(data)) def resume(jid, state_id=None): @@ -324,7 +324,7 @@ def resume(jid, state_id=None): if state_id == '__all__': data = {} with salt.utils.files.fopen(pause_path, 'wb') as fp_: - fp_.write(msgpack.dumps(data)) + fp_.write(salt.utils.msgpack.dumps(data)) def orchestrate(mods, diff --git a/salt/modules/win_repo.py b/salt/modules/win_repo.py index c0a9dde4536d..21f35164df07 100644 --- a/salt/modules/win_repo.py +++ b/salt/modules/win_repo.py @@ -31,11 +31,8 @@ PER_REMOTE_ONLY ) from salt.ext import six -try: - import msgpack -except ImportError: - import msgpack_pure as msgpack # pylint: disable=import-error import salt.utils.gitfs +import salt.utils.msgpack # pylint: enable=unused-import log = logging.getLogger(__name__) diff --git a/salt/payload.py b/salt/payload.py index 5df6458b861a..aa6b8f8792d9 100644 --- a/salt/payload.py +++ b/salt/payload.py @@ -55,6 +55,10 @@ #sys.exit(salt.defaults.exitcodes.EX_GENERIC) +if HAS_MSGPACK: + import salt.utils.msgpack + + if HAS_MSGPACK and not hasattr(msgpack, 'exceptions'): class PackValueError(Exception): ''' @@ -75,14 +79,15 @@ def package(payload): This method for now just wraps msgpack.dumps, but it is here so that we can make the serialization a custom option in the future with ease. ''' - return msgpack.dumps(payload) + return salt.utils.msgpack.dumps(payload, _msgpack_module=msgpack) def unpackage(package_): ''' Unpackages a payload ''' - return msgpack.loads(package_, use_list=True) + return salt.utils.msgpack.loads(package_, use_list=True, + _msgpack_module=msgpack) def format_payload(enc, **kwargs): @@ -142,12 +147,17 @@ def ext_type_decoder(code, data): # that under Python 2 we can still work with older versions # of msgpack. try: - ret = msgpack.loads(msg, use_list=True, ext_hook=ext_type_decoder, encoding=encoding) + ret = salt.utils.msgpack.loads(msg, use_list=True, + ext_hook=ext_type_decoder, + encoding=encoding, + _msgpack_module=msgpack) except UnicodeDecodeError: # msg contains binary data ret = msgpack.loads(msg, use_list=True, ext_hook=ext_type_decoder) else: - ret = msgpack.loads(msg, use_list=True, ext_hook=ext_type_decoder) + ret = salt.utils.msgpack.loads(msg, use_list=True, + ext_hook=ext_type_decoder, + _msgpack_module=msgpack) if six.PY3 and encoding is None and not raw: ret = salt.transport.frame.decode_embedded_strs(ret) except Exception as exc: @@ -214,9 +224,12 @@ def ext_type_encoder(obj): # Due to this, if we don't need it, don't pass it at all so # that under Python 2 we can still work with older versions # of msgpack. - return msgpack.dumps(msg, default=ext_type_encoder, use_bin_type=use_bin_type) + return salt.utils.msgpack.dumps(msg, default=ext_type_encoder, + use_bin_type=use_bin_type, + _msgpack_module=msgpack) else: - return msgpack.dumps(msg, default=ext_type_encoder) + return salt.utils.msgpack.dumps(msg, default=ext_type_encoder, + _msgpack_module=msgpack) except (OverflowError, msgpack.exceptions.PackValueError): # msgpack<=0.4.6 don't call ext encoder on very long integers raising the error instead. # Convert any very long longs to strings and call dumps again. @@ -239,9 +252,12 @@ def verylong_encoder(obj): msg = verylong_encoder(msg) if msgpack.version >= (0, 4, 0): - return msgpack.dumps(msg, default=ext_type_encoder, use_bin_type=use_bin_type) + return salt.utils.msgpack.dumps(msg, default=ext_type_encoder, + use_bin_type=use_bin_type, + _msgpack_module=msgpack) else: - return msgpack.dumps(msg, default=ext_type_encoder) + return salt.utils.msgpack.dumps(msg, default=ext_type_encoder, + _msgpack_module=msgpack) def dump(self, msg, fn_): ''' diff --git a/salt/renderers/msgpack.py b/salt/renderers/msgpack.py index f58d11b85b8d..eceac4f53bb5 100644 --- a/salt/renderers/msgpack.py +++ b/salt/renderers/msgpack.py @@ -1,10 +1,8 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals -# Import third party libs -import msgpack - # Import salt libs +import salt.utils.msgpack from salt.ext import six @@ -28,4 +26,4 @@ def render(msgpack_data, saltenv='base', sls='', **kws): msgpack_data = msgpack_data[(msgpack_data.find('\n') + 1):] if not msgpack_data.strip(): return {} - return msgpack.loads(msgpack_data) + return salt.utils.msgpack.loads(msgpack_data) diff --git a/salt/returners/local_cache.py b/salt/returners/local_cache.py index 6799730877be..721bd70045e6 100644 --- a/salt/returners/local_cache.py +++ b/salt/returners/local_cache.py @@ -20,11 +20,11 @@ import salt.utils.files import salt.utils.jid import salt.utils.minions +import salt.utils.msgpack import salt.utils.stringutils import salt.exceptions # Import 3rd-party libs -import msgpack from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin @@ -514,7 +514,7 @@ def save_reg(data): raise try: with salt.utils.files.fopen(regfile, 'a') as fh_: - msgpack.dump(data, fh_) + salt.utils.msgpack.dump(data, fh_) except: log.error('Could not write to msgpack file %s', __opts__['outdir']) raise @@ -528,7 +528,7 @@ def load_reg(): regfile = os.path.join(reg_dir, 'register') try: with salt.utils.files.fopen(regfile, 'r') as fh_: - return msgpack.load(fh_) + return salt.utils.msgpack.load(fh_) except: log.error('Could not write to msgpack file %s', __opts__['outdir']) raise diff --git a/salt/runners/winrepo.py b/salt/runners/winrepo.py index 480a3138b66b..321b91638eb8 100644 --- a/salt/runners/winrepo.py +++ b/salt/runners/winrepo.py @@ -12,15 +12,12 @@ # Import third party libs from salt.ext import six -try: - import msgpack -except ImportError: - import msgpack_pure as msgpack # pylint: disable=import-error # Import salt libs from salt.exceptions import CommandExecutionError, SaltRenderError import salt.utils.files import salt.utils.gitfs +import salt.utils.msgpack import salt.utils.path import logging import salt.minion @@ -124,7 +121,7 @@ def genrepo(opts=None, fire_event=True): ret.setdefault('name_map', {}).update(revmap) with salt.utils.files.fopen( os.path.join(winrepo_dir, winrepo_cachefile), 'w+b') as repo: - repo.write(msgpack.dumps(ret)) + repo.write(salt.utils.msgpack.dumps(ret)) return ret diff --git a/salt/sdb/sqlite3.py b/salt/sdb/sqlite3.py index 540a289d56ae..006d574c283b 100644 --- a/salt/sdb/sqlite3.py +++ b/salt/sdb/sqlite3.py @@ -54,11 +54,9 @@ HAS_SQLITE3 = False # Import salt libs +import salt.utils.msgpack from salt.ext import six -# Import third party libs -import msgpack - DEFAULT_TABLE = 'sdb' @@ -126,9 +124,9 @@ def set_(key, value, profile=None): return False conn, cur, table = _connect(profile) if six.PY2: - value = buffer(msgpack.packb(value)) + value = buffer(salt.utils.msgpack.packb(value)) else: - value = memoryview(msgpack.packb(value)) + value = memoryview(salt.utils.msgpack.packb(value)) q = profile.get('set_query', ('INSERT OR REPLACE INTO {0} VALUES ' '(:key, :value)').format(table)) conn.execute(q, {'key': key, 'value': value}) @@ -149,4 +147,4 @@ def get(key, profile=None): res = res.fetchone() if not res: return None - return msgpack.unpackb(res[0]) + return salt.utils.msgpack.unpackb(res[0]) diff --git a/salt/serializers/msgpack.py b/salt/serializers/msgpack.py index f55fa878b669..90df192ee3c6 100644 --- a/salt/serializers/msgpack.py +++ b/salt/serializers/msgpack.py @@ -24,6 +24,7 @@ try: # Attempt to import msgpack import msgpack + import salt.utils.msgpack # There is a serialization issue on ARM and potentially other platforms # for some msgpack bindings, check for it if msgpack.loads(msgpack.dumps([1, 2, 3]), use_list=True) is None: @@ -33,6 +34,7 @@ # Fall back to msgpack_pure try: import msgpack_pure as msgpack # pylint: disable=import-error + import salt.utils.msgpack except ImportError: # TODO: Come up with a sane way to get a configured logfile # and write to the logfile when this error is hit also @@ -60,7 +62,8 @@ def _deserialize(stream_or_string, **options): def _serialize(obj, **options): try: - return msgpack.dumps(obj, **options) + return salt.utils.msgpack.dumps(obj, _msgpack_module=msgpack, + **options) except Exception as error: raise SerializationError(error) @@ -68,7 +71,9 @@ def _deserialize(stream_or_string, **options): try: options.setdefault('use_list', True) options.setdefault('encoding', 'utf-8') - return msgpack.loads(stream_or_string, **options) + return salt.utils.msgpack.loads(stream_or_string, + _msgpack_module=msgpack, + **options) except Exception as error: raise DeserializationError(error) @@ -95,14 +100,16 @@ def _decoder(obj): def _serialize(obj, **options): try: obj = _encoder(obj) - return msgpack.dumps(obj, **options) + return salt.utils.msgpack.dumps(obj, _msgpack_module=msgpack, + **options) except Exception as error: raise SerializationError(error) def _deserialize(stream_or_string, **options): options.setdefault('use_list', True) try: - obj = msgpack.loads(stream_or_string) + obj = salt.utils.msgpack.loads(stream_or_string, + _msgpack_module=msgpack) return _decoder(obj) except Exception as error: raise DeserializationError(error) diff --git a/salt/state.py b/salt/state.py index 80f40f5e6711..be1821093311 100644 --- a/salt/state.py +++ b/salt/state.py @@ -38,6 +38,7 @@ import salt.utils.event import salt.utils.files import salt.utils.immutabletypes as immutabletypes +import salt.utils.msgpack as msgpack import salt.utils.platform import salt.utils.process import salt.utils.url @@ -54,7 +55,6 @@ import salt.utils.yamlloader as yamlloader # Import third party libs -import msgpack # pylint: disable=import-error,no-name-in-module,redefined-builtin from salt.ext import six from salt.ext.six.moves import map, range, reload_module diff --git a/salt/states/netsnmp.py b/salt/states/netsnmp.py index 9bafffefbc93..7fcdf6fc4132 100644 --- a/salt/states/netsnmp.py +++ b/salt/states/netsnmp.py @@ -23,9 +23,8 @@ import logging log = logging.getLogger(__name__) -from json import loads, dumps - # salt lib +from salt.utils.json import loads, dumps from salt.ext import six # import NAPALM utils import salt.utils.napalm diff --git a/salt/states/netusers.py b/salt/states/netusers.py index 58b3751c6b74..f4ff648fbfdd 100644 --- a/salt/states/netusers.py +++ b/salt/states/netusers.py @@ -25,9 +25,9 @@ # Python std lib from copy import deepcopy -from json import loads, dumps # salt lib +from salt.utils.json import loads, dumps from salt.ext import six # import NAPALM utils import salt.utils.napalm diff --git a/salt/states/pkg.py b/salt/states/pkg.py index eed114d9cec4..07178871aab6 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -135,10 +135,6 @@ # The following imports are used by the namespaced win_pkg funcs # and need to be included in their globals. # pylint: disable=import-error,unused-import - try: - import msgpack - except ImportError: - import msgpack_pure as msgpack from salt.utils.versions import LooseVersion # pylint: enable=import-error,unused-import # pylint: enable=invalid-name diff --git a/salt/states/probes.py b/salt/states/probes.py index e621fb353606..f79eeb71d027 100644 --- a/salt/states/probes.py +++ b/salt/states/probes.py @@ -25,9 +25,9 @@ log = logging.getLogger(__name__) from copy import deepcopy -from json import loads, dumps # salt modules +from salt.utils.json import loads, dumps from salt.ext import six # import NAPALM utils import salt.utils.napalm diff --git a/salt/states/zabbix_host.py b/salt/states/zabbix_host.py index 85f0b322efda..85e173bf2f71 100644 --- a/salt/states/zabbix_host.py +++ b/salt/states/zabbix_host.py @@ -7,7 +7,7 @@ ''' from __future__ import absolute_import, print_function, unicode_literals -from json import loads, dumps +from salt.utils.json import loads, dumps from copy import deepcopy from salt.ext import six diff --git a/salt/states/zabbix_user.py b/salt/states/zabbix_user.py index 519d236290ad..e6dd2eb90c2e 100644 --- a/salt/states/zabbix_user.py +++ b/salt/states/zabbix_user.py @@ -9,7 +9,7 @@ # Import Python libs from __future__ import absolute_import, print_function, unicode_literals -from json import loads, dumps +from salt.utils.json import loads, dumps from copy import deepcopy # Import Salt libs diff --git a/salt/transport/frame.py b/salt/transport/frame.py index 33d0c0d91703..88b595184ec7 100644 --- a/salt/transport/frame.py +++ b/salt/transport/frame.py @@ -4,7 +4,7 @@ ''' # Import python libs from __future__ import absolute_import, print_function, unicode_literals -import msgpack +import salt.utils.msgpack from salt.ext import six @@ -18,7 +18,7 @@ def frame_msg(body, header=None, raw_body=False): # pylint: disable=unused-argu framed_msg['head'] = header framed_msg['body'] = body - return msgpack.dumps(framed_msg) + return salt.utils.msgpack.dumps(framed_msg) def frame_msg_ipc(body, header=None, raw_body=False): # pylint: disable=unused-argument @@ -35,9 +35,9 @@ def frame_msg_ipc(body, header=None, raw_body=False): # pylint: disable=unused- framed_msg['head'] = header framed_msg['body'] = body if six.PY2: - return msgpack.dumps(framed_msg) + return salt.utils.msgpack.dumps(framed_msg) else: - return msgpack.dumps(framed_msg, use_bin_type=True) + return salt.utils.msgpack.dumps(framed_msg, use_bin_type=True) def _decode_embedded_list(src): diff --git a/salt/transport/ipc.py b/salt/transport/ipc.py index f1f844bca02b..84f87928b71b 100644 --- a/salt/transport/ipc.py +++ b/salt/transport/ipc.py @@ -10,9 +10,6 @@ import weakref import time -# Import 3rd-party libs -import msgpack - # Import Tornado libs import tornado import tornado.gen @@ -24,6 +21,7 @@ # Import Salt libs import salt.transport.client import salt.transport.frame +import salt.utils.msgpack as msgpack from salt.ext import six log = logging.getLogger(__name__) diff --git a/salt/transport/tcp.py b/salt/transport/tcp.py index 598ea07a8b5d..9450ee0ad93e 100644 --- a/salt/transport/tcp.py +++ b/salt/transport/tcp.py @@ -10,7 +10,6 @@ from __future__ import absolute_import, print_function, unicode_literals import errno import logging -import msgpack import socket import os import weakref @@ -22,6 +21,7 @@ import salt.utils.asynchronous import salt.utils.event import salt.utils.files +import salt.utils.msgpack as msgpack import salt.utils.platform import salt.utils.process import salt.utils.verify diff --git a/salt/utils/cache.py b/salt/utils/cache.py index 6581af9f870d..adf34fb8f074 100644 --- a/salt/utils/cache.py +++ b/salt/utils/cache.py @@ -9,7 +9,7 @@ import time import logging try: - import msgpack + import salt.utils.msgpack HAS_MSGPACK = True except ImportError: HAS_MSGPACK = False @@ -139,7 +139,9 @@ def _read(self): if not HAS_MSGPACK or not os.path.exists(self._path): return with salt.utils.files.fopen(self._path, 'rb') as fp_: - cache = salt.utils.data.decode(msgpack.load(fp_, encoding=__salt_system_encoding__)) + cache = salt.utils.data.decode( + salt.utils.msgpack.load( + fp_, encoding=__salt_system_encoding__)) if "CacheDisk_cachetime" in cache: # new format self._dict = cache["CacheDisk_data"] self._key_cache_time = cache["CacheDisk_cachetime"] @@ -164,7 +166,7 @@ def _write(self): "CacheDisk_data": self._dict, "CacheDisk_cachetime": self._key_cache_time } - msgpack.dump(cache, fp_, use_bin_type=True) + salt.utils.msgpack.dump(cache, fp_, use_bin_type=True) class CacheCli(object): diff --git a/salt/utils/cloud.py b/salt/utils/cloud.py index c1ee50afa509..64dff483f5e1 100644 --- a/salt/utils/cloud.py +++ b/salt/utils/cloud.py @@ -18,7 +18,6 @@ import multiprocessing import logging import pipes -import msgpack import traceback import copy import re @@ -51,6 +50,7 @@ import salt.utils.data import salt.utils.event import salt.utils.files +import salt.utils.msgpack import salt.utils.platform import salt.utils.stringutils import salt.utils.versions @@ -2510,7 +2510,9 @@ def cachedir_index_add(minion_id, profile, driver, provider, base=None): if os.path.exists(index_file): mode = 'rb' if six.PY3 else 'r' with salt.utils.files.fopen(index_file, mode) as fh_: - index = salt.utils.data.decode(msgpack.load(fh_, encoding=MSGPACK_ENCODING)) + index = salt.utils.data.decode( + salt.utils.msgpack.msgpack.load( + fh_, encoding=MSGPACK_ENCODING)) else: index = {} @@ -2527,7 +2529,7 @@ def cachedir_index_add(minion_id, profile, driver, provider, base=None): mode = 'wb' if six.PY3 else 'w' with salt.utils.files.fopen(index_file, mode) as fh_: - msgpack.dump(index, fh_, encoding=MSGPACK_ENCODING) + salt.utils.msgpack.dump(index, fh_, encoding=MSGPACK_ENCODING) unlock_file(index_file) @@ -2544,7 +2546,8 @@ def cachedir_index_del(minion_id, base=None): if os.path.exists(index_file): mode = 'rb' if six.PY3 else 'r' with salt.utils.files.fopen(index_file, mode) as fh_: - index = salt.utils.data.decode(msgpack.load(fh_, encoding=MSGPACK_ENCODING)) + index = salt.utils.data.decode( + salt.utils.msgpack.load(fh_, encoding=MSGPACK_ENCODING)) else: return @@ -2553,7 +2556,7 @@ def cachedir_index_del(minion_id, base=None): mode = 'wb' if six.PY3 else 'w' with salt.utils.files.fopen(index_file, mode) as fh_: - msgpack.dump(index, fh_, encoding=MSGPACK_ENCODING) + salt.utils.msgpack.dump(index, fh_, encoding=MSGPACK_ENCODING) unlock_file(index_file) @@ -2611,7 +2614,7 @@ def request_minion_cachedir( path = os.path.join(base, 'requested', fname) mode = 'wb' if six.PY3 else 'w' with salt.utils.files.fopen(path, mode) as fh_: - msgpack.dump(data, fh_, encoding=MSGPACK_ENCODING) + salt.utils.msgpack.dump(data, fh_, encoding=MSGPACK_ENCODING) def change_minion_cachedir( @@ -2643,12 +2646,13 @@ def change_minion_cachedir( path = os.path.join(base, cachedir, fname) with salt.utils.files.fopen(path, 'r') as fh_: - cache_data = salt.utils.data.decode(msgpack.load(fh_, encoding=MSGPACK_ENCODING)) + cache_data = salt.utils.data.decode( + salt.utils.msgpack.load(fh_, encoding=MSGPACK_ENCODING)) cache_data.update(data) with salt.utils.files.fopen(path, 'w') as fh_: - msgpack.dump(cache_data, fh_, encoding=MSGPACK_ENCODING) + salt.utils.msgpack.dump(cache_data, fh_, encoding=MSGPACK_ENCODING) def activate_minion_cachedir(minion_id, base=None): @@ -2722,7 +2726,8 @@ def list_cache_nodes_full(opts=None, provider=None, base=None): minion_id = fname[:-2] # strip '.p' from end of msgpack filename mode = 'rb' if six.PY3 else 'r' with salt.utils.files.fopen(fpath, mode) as fh_: - minions[driver][prov][minion_id] = salt.utils.data.decode(msgpack.load(fh_, encoding=MSGPACK_ENCODING)) + minions[driver][prov][minion_id] = salt.utils.data.decode( + salt.utils.msgpack.load(fh_, encoding=MSGPACK_ENCODING)) return minions @@ -2899,7 +2904,7 @@ def cache_node_list(nodes, provider, opts): path = os.path.join(prov_dir, '{0}.p'.format(node)) mode = 'wb' if six.PY3 else 'w' with salt.utils.files.fopen(path, mode) as fh_: - msgpack.dump(nodes[node], fh_, encoding=MSGPACK_ENCODING) + salt.utils.msgpack.dump(nodes[node], fh_, encoding=MSGPACK_ENCODING) def cache_node(node, provider, opts): @@ -2925,7 +2930,7 @@ def cache_node(node, provider, opts): path = os.path.join(prov_dir, '{0}.p'.format(node['name'])) mode = 'wb' if six.PY3 else 'w' with salt.utils.files.fopen(path, mode) as fh_: - msgpack.dump(node, fh_, encoding=MSGPACK_ENCODING) + salt.utils.msgpack.dump(node, fh_, encoding=MSGPACK_ENCODING) def missing_node_cache(prov_dir, node_list, provider, opts): @@ -3000,7 +3005,8 @@ def diff_node_cache(prov_dir, node, new_data, opts): with salt.utils.files.fopen(path, 'r') as fh_: try: - cache_data = salt.utils.data.decode(msgpack.load(fh_, encoding=MSGPACK_ENCODING)) + cache_data = salt.utils.data.decode( + salt.utils.msgpack.load(fh_, encoding=MSGPACK_ENCODING)) except ValueError: log.warning('Cache for %s was corrupt: Deleting', node) cache_data = {} diff --git a/salt/utils/http.py b/salt/utils/http.py index ee511d37fed4..97d25db9ba99 100644 --- a/salt/utils/http.py +++ b/salt/utils/http.py @@ -83,7 +83,7 @@ HAS_REQUESTS = False try: - import msgpack + import salt.utils.msgpack HAS_MSGPACK = True except ImportError: HAS_MSGPACK = False @@ -274,12 +274,12 @@ def query(url, # contain expirations, they can't be stored in a proper cookie jar. if os.path.isfile(session_cookie_jar): with salt.utils.files.fopen(session_cookie_jar, 'rb') as fh_: - session_cookies = msgpack.load(fh_) + session_cookies = salt.utils.msgpack.load(fh_) if isinstance(session_cookies, dict): header_dict.update(session_cookies) else: with salt.utils.files.fopen(session_cookie_jar, 'wb') as fh_: - msgpack.dump('', fh_) + salt.utils.msgpack.dump('', fh_) for header in header_list: comps = header.split(':') @@ -630,9 +630,9 @@ def query(url, with salt.utils.files.fopen(session_cookie_jar, 'wb') as fh_: session_cookies = result_headers.get('set-cookie', None) if session_cookies is not None: - msgpack.dump({'Cookie': session_cookies}, fh_) + salt.utils.msgpack.dump({'Cookie': session_cookies}, fh_) else: - msgpack.dump('', fh_) + salt.utils.msgpack.dump('', fh_) if status is True: ret['status'] = result_status_code diff --git a/salt/utils/msgpack.py b/salt/utils/msgpack.py new file mode 100644 index 000000000000..7e66cb8ed739 --- /dev/null +++ b/salt/utils/msgpack.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +''' +Functions to work with MessagePack +''' + +from __future__ import absolute_import + +# Import Python libs +try: + # Attempt to import msgpack + import msgpack +except ImportError: + # Fall back to msgpack_pure + import msgpack_pure as msgpack # pylint: disable=import-error + + +def pack(o, stream, **kwargs): + ''' + .. versionadded:: 2018.3.4 + + Wraps msgpack.pack and ensures that the passed object is unwrapped if it is + a proxy. + + By default, this function uses the msgpack module and falls back to + msgpack_pure, if the msgpack is not available. You can pass an alternate + msgpack module using the _msgpack_module argument. + ''' + msgpack_module = kwargs.pop('_msgpack_module', msgpack) + return msgpack_module.pack(o, stream, **kwargs) + + +def packb(o, **kwargs): + ''' + .. versionadded:: 2018.3.4 + + Wraps msgpack.packb and ensures that the passed object is unwrapped if it + is a proxy. + + By default, this function uses the msgpack module and falls back to + msgpack_pure, if the msgpack is not available. You can pass an alternate + msgpack module using the _msgpack_module argument. + ''' + msgpack_module = kwargs.pop('_msgpack_module', msgpack) + return msgpack_module.packb(o, **kwargs) + + +def unpack(stream, **kwargs): + ''' + .. versionadded:: 2018.3.4 + + Wraps msgpack.unpack. + + By default, this function uses the msgpack module and falls back to + msgpack_pure, if the msgpack is not available. You can pass an alternate + msgpack module using the _msgpack_module argument. + ''' + msgpack_module = kwargs.pop('_msgpack_module', msgpack) + return msgpack_module.unpack(stream, **kwargs) + + +def unpackb(packed, **kwargs): + ''' + .. versionadded:: 2018.3.4 + + Wraps msgpack.unpack. + + By default, this function uses the msgpack module and falls back to + msgpack_pure, if the msgpack is not available. You can pass an alternate + msgpack module using the _msgpack_module argument. + ''' + msgpack_module = kwargs.pop('_msgpack_module', msgpack) + return msgpack_module.unpackb(packed, **kwargs) + + +# alias for compatibility to simplejson/marshal/pickle. +load = unpack +loads = unpackb + +dump = pack +dumps = packb diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index e706802aa74c..3373f87602cb 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -50,6 +50,7 @@ import salt.version import salt.utils.color import salt.utils.files +import salt.utils.msgpack as msgpack import salt.utils.path import salt.utils.platform import salt.utils.process @@ -68,7 +69,6 @@ pass # Import 3rd-party libs -import msgpack from salt.ext import six from salt.ext.six.moves import cStringIO diff --git a/tests/integration/files/log_handlers/runtests_log_handler.py b/tests/integration/files/log_handlers/runtests_log_handler.py index 983447d97ae6..2269c063ff83 100644 --- a/tests/integration/files/log_handlers/runtests_log_handler.py +++ b/tests/integration/files/log_handlers/runtests_log_handler.py @@ -19,10 +19,8 @@ import threading from multiprocessing import Queue -# Import 3rd-party libs -import msgpack - # Import Salt libs +import salt.utils.msgpack from salt.ext import six import salt.log.setup @@ -85,7 +83,8 @@ def process_queue(port, queue): break # Just log everything, filtering will happen on the main process # logging handlers - sock.sendall(msgpack.dumps(record.__dict__, encoding='utf-8')) + sock.sendall(salt.utils.msgpack.dumps(record.__dict__, + encoding='utf-8')) except (IOError, EOFError, KeyboardInterrupt, SystemExit): sock.shutdown(socket.SHUT_RDWR) sock.close() diff --git a/tests/packdump.py b/tests/packdump.py index 92ed79de29bc..5a230eed946f 100644 --- a/tests/packdump.py +++ b/tests/packdump.py @@ -9,8 +9,8 @@ import sys import pprint -# Import third party libs -import msgpack +# Import Salt libs +import salt.utils.msgpack def dump(path): @@ -21,7 +21,7 @@ def dump(path): print('Not a file') return with open(path, 'rb') as fp_: - data = msgpack.loads(fp_.read()) + data = salt.utils.msgpack.loads(fp_.read()) pprint.pprint(data)