From 6f675f711a807af61233355c6d127a3a2e412dbe Mon Sep 17 00:00:00 2001 From: Riccardo Schirone Date: Wed, 26 Feb 2020 19:03:09 +0100 Subject: [PATCH 1/3] Prevents arbitrary code execution during python/object/new constructor In FullLoader python/object/new constructor, implemented by construct_python_object_apply, has support for setting the state of a deserialized instance through the set_python_instance_state method. After setting the state, some operations are performed on the instance to complete its initialization, however it is possible for an attacker to set the instance' state in such a way that arbitrary code is executed by the FullLoader. This patch tries to block such attacks in FullLoader by preventing set_python_instance_state from setting arbitrary properties. It implements a blacklist that includes `extend` method (called by construct_python_object_apply) and all special methods (e.g. __set__, __setitem__, etc.). Users who need special attributes being set in the state of a deserialized object can still do it through the UnsafeLoader, which however should not be used on untrusted input. Additionally, they can subclass FullLoader and redefine `get_state_keys_blacklist()` to extend/replace the list of blacklisted keys, passing the subclassed loader to yaml.load. --- lib/yaml/constructor.py | 29 ++++++++++++++++++++++++++++- lib3/yaml/constructor.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/lib/yaml/constructor.py b/lib/yaml/constructor.py index 8ce39722..794681cb 100644 --- a/lib/yaml/constructor.py +++ b/lib/yaml/constructor.py @@ -56,6 +56,14 @@ def check_data(self): # If there are more documents available? return self.check_node() + def check_state_key(self, key): + """Block special attributes/methods from being set in a newly created + object, to prevent user-controlled methods from being called during + deserialization""" + if self.get_state_keys_blacklist_regexp().match(key): + raise ConstructorError(None, None, + "blacklisted key '%s' in instance state found" % (key,), None) + def get_data(self): # Construct and return the next document. if self.check_node(): @@ -495,6 +503,16 @@ def construct_undefined(self, node): SafeConstructor.construct_undefined) class FullConstructor(SafeConstructor): + # 'extend' is blacklisted because it is used by + # construct_python_object_apply to add `listitems` to a newly generate + # python instance + def get_state_keys_blacklist(self): + return ['^extend$', '^__.*__$'] + + def get_state_keys_blacklist_regexp(self): + if not hasattr(self, 'state_keys_blacklist_regexp'): + self.state_keys_blacklist_regexp = re.compile('(' + '|'.join(self.get_state_keys_blacklist()) + ')') + return self.state_keys_blacklist_regexp def construct_python_str(self, node): return self.construct_scalar(node).encode('utf-8') @@ -590,7 +608,7 @@ def make_python_instance(self, suffix, node, else: return cls(*args, **kwds) - def set_python_instance_state(self, instance, state): + def set_python_instance_state(self, instance, state, unsafe=False): if hasattr(instance, '__setstate__'): instance.__setstate__(state) else: @@ -598,10 +616,15 @@ def set_python_instance_state(self, instance, state): if isinstance(state, tuple) and len(state) == 2: state, slotstate = state if hasattr(instance, '__dict__'): + if not unsafe and state: + for key in state.keys(): + self.check_state_key(key) instance.__dict__.update(state) elif state: slotstate.update(state) for key, value in slotstate.items(): + if not unsafe: + self.check_state_key(key) setattr(instance, key, value) def construct_python_object(self, suffix, node): @@ -723,6 +746,10 @@ def make_python_instance(self, suffix, node, args=None, kwds=None, newobj=False) return super(UnsafeConstructor, self).make_python_instance( suffix, node, args, kwds, newobj, unsafe=True) + def set_python_instance_state(self, instance, state): + return super(UnsafeConstructor, self).set_python_instance_state( + instance, state, unsafe=True) + UnsafeConstructor.add_multi_constructor( u'tag:yaml.org,2002:python/object/apply:', UnsafeConstructor.construct_python_object_apply) diff --git a/lib3/yaml/constructor.py b/lib3/yaml/constructor.py index cd9167ea..1948b125 100644 --- a/lib3/yaml/constructor.py +++ b/lib3/yaml/constructor.py @@ -31,6 +31,14 @@ def check_data(self): # If there are more documents available? return self.check_node() + def check_state_key(self, key): + """Block special attributes/methods from being set in a newly created + object, to prevent user-controlled methods from being called during + deserialization""" + if self.get_state_keys_blacklist_regexp().match(key): + raise ConstructorError(None, None, + "blacklisted key '%s' in instance state found" % (key,), None) + def get_data(self): # Construct and return the next document. if self.check_node(): @@ -472,6 +480,16 @@ def construct_undefined(self, node): SafeConstructor.construct_undefined) class FullConstructor(SafeConstructor): + # 'extend' is blacklisted because it is used by + # construct_python_object_apply to add `listitems` to a newly generate + # python instance + def get_state_keys_blacklist(self): + return ['^extend$', '^__.*__$'] + + def get_state_keys_blacklist_regexp(self): + if not hasattr(self, 'state_keys_blacklist_regexp'): + self.state_keys_blacklist_regexp = re.compile('(' + '|'.join(self.get_state_keys_blacklist()) + ')') + return self.state_keys_blacklist_regexp def construct_python_str(self, node): return self.construct_scalar(node) @@ -574,7 +592,7 @@ def make_python_instance(self, suffix, node, else: return cls(*args, **kwds) - def set_python_instance_state(self, instance, state): + def set_python_instance_state(self, instance, state, unsafe=False): if hasattr(instance, '__setstate__'): instance.__setstate__(state) else: @@ -582,10 +600,15 @@ def set_python_instance_state(self, instance, state): if isinstance(state, tuple) and len(state) == 2: state, slotstate = state if hasattr(instance, '__dict__'): + if not unsafe and state: + for key in state.keys(): + self.check_state_key(key) instance.__dict__.update(state) elif state: slotstate.update(state) for key, value in slotstate.items(): + if not unsafe: + self.check_state_key(key) setattr(instance, key, value) def construct_python_object(self, suffix, node): @@ -711,6 +734,10 @@ def make_python_instance(self, suffix, node, args=None, kwds=None, newobj=False) return super(UnsafeConstructor, self).make_python_instance( suffix, node, args, kwds, newobj, unsafe=True) + def set_python_instance_state(self, instance, state): + return super(UnsafeConstructor, self).set_python_instance_state( + instance, state, unsafe=True) + UnsafeConstructor.add_multi_constructor( 'tag:yaml.org,2002:python/object/apply:', UnsafeConstructor.construct_python_object_apply) From a946ac6ef1bae466062e7c70f86e17d019bcd65c Mon Sep 17 00:00:00 2001 From: Riccardo Schirone Date: Thu, 5 Mar 2020 16:55:21 +0100 Subject: [PATCH 2/3] Make sure python/object/new constructor does not set some properties --- tests/data/overwrite-state-new-constructor.loader-error | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 tests/data/overwrite-state-new-constructor.loader-error diff --git a/tests/data/overwrite-state-new-constructor.loader-error b/tests/data/overwrite-state-new-constructor.loader-error new file mode 100644 index 00000000..8d224f15 --- /dev/null +++ b/tests/data/overwrite-state-new-constructor.loader-error @@ -0,0 +1,5 @@ +- !!python/object/new:yaml.MappingNode + args: + state: + extend: test + __test__: test From 90915656066489dc0f4db9f5d1b8bac686d05bc5 Mon Sep 17 00:00:00 2001 From: Riccardo Schirone Date: Tue, 10 Mar 2020 10:20:35 +0100 Subject: [PATCH 3/3] Add test to show how to subclass FullLoader with new blacklist --- tests/data/myfullloader.subclass_blacklist | 5 +++++ tests/lib/test_constructor.py | 18 +++++++++++++++++- tests/lib3/test_constructor.py | 18 +++++++++++++++++- 3 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 tests/data/myfullloader.subclass_blacklist diff --git a/tests/data/myfullloader.subclass_blacklist b/tests/data/myfullloader.subclass_blacklist new file mode 100644 index 00000000..555a2b3a --- /dev/null +++ b/tests/data/myfullloader.subclass_blacklist @@ -0,0 +1,5 @@ +- !!python/object/new:yaml.MappingNode + args: + state: + mymethod: test + wrong_method: test2 diff --git a/tests/lib/test_constructor.py b/tests/lib/test_constructor.py index a22fd189..5a8cce21 100644 --- a/tests/lib/test_constructor.py +++ b/tests/lib/test_constructor.py @@ -17,7 +17,7 @@ def _make_objects(): global MyLoader, MyDumper, MyTestClass1, MyTestClass2, MyTestClass3, YAMLObject1, YAMLObject2, \ AnObject, AnInstance, AState, ACustomState, InitArgs, InitArgsWithState, \ NewArgs, NewArgsWithState, Reduce, ReduceWithState, Slots, MyInt, MyList, MyDict, \ - FixedOffset, today, execute + FixedOffset, today, execute, MyFullLoader class MyLoader(yaml.Loader): pass @@ -235,6 +235,10 @@ def tzname(self, dt): def dst(self, dt): return datetime.timedelta(0) + class MyFullLoader(yaml.FullLoader): + def get_state_keys_blacklist(self): + return super(MyFullLoader, self).get_state_keys_blacklist() + ['^mymethod$', '^wrong_.*$'] + today = datetime.date.today() def _load_code(expression): @@ -289,6 +293,18 @@ def test_constructor_types(data_filename, code_filename, verbose=False): test_constructor_types.unittest = ['.data', '.code'] +def test_subclass_blacklist_types(data_filename, verbose=False): + _make_objects() + try: + yaml.load(open(data_filename, 'rb').read(), MyFullLoader) + except yaml.YAMLError as exc: + if verbose: + print("%s:" % exc.__class__.__name__, exc) + else: + raise AssertionError("expected an exception") + +test_subclass_blacklist_types.unittest = ['.subclass_blacklist'] + if __name__ == '__main__': import sys, test_constructor sys.modules['test_constructor'] = sys.modules['__main__'] diff --git a/tests/lib3/test_constructor.py b/tests/lib3/test_constructor.py index 877982db..f9a50770 100644 --- a/tests/lib3/test_constructor.py +++ b/tests/lib3/test_constructor.py @@ -14,7 +14,7 @@ def _make_objects(): global MyLoader, MyDumper, MyTestClass1, MyTestClass2, MyTestClass3, YAMLObject1, YAMLObject2, \ AnObject, AnInstance, AState, ACustomState, InitArgs, InitArgsWithState, \ NewArgs, NewArgsWithState, Reduce, ReduceWithState, Slots, MyInt, MyList, MyDict, \ - FixedOffset, today, execute + FixedOffset, today, execute, MyFullLoader class MyLoader(yaml.Loader): pass @@ -222,6 +222,10 @@ def tzname(self, dt): def dst(self, dt): return datetime.timedelta(0) + class MyFullLoader(yaml.FullLoader): + def get_state_keys_blacklist(self): + return super().get_state_keys_blacklist() + ['^mymethod$', '^wrong_.*$'] + today = datetime.date.today() def _load_code(expression): @@ -274,6 +278,18 @@ def test_constructor_types(data_filename, code_filename, verbose=False): test_constructor_types.unittest = ['.data', '.code'] +def test_subclass_blacklist_types(data_filename, verbose=False): + _make_objects() + try: + yaml.load(open(data_filename, 'rb').read(), MyFullLoader) + except yaml.YAMLError as exc: + if verbose: + print("%s:" % exc.__class__.__name__, exc) + else: + raise AssertionError("expected an exception") + +test_subclass_blacklist_types.unittest = ['.subclass_blacklist'] + if __name__ == '__main__': import sys, test_constructor sys.modules['test_constructor'] = sys.modules['__main__']