From 59eec9a4ee053da0326c0c72288c7dabc6f692b7 Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Thu, 4 Feb 2021 12:52:30 -0700 Subject: [PATCH 01/24] install_requires --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index d7476c61..4fe6afcc 100644 --- a/setup.py +++ b/setup.py @@ -293,4 +293,5 @@ def run(self): distclass=Distribution, cmdclass=cmdclass, python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', + install_requires: ['requests'], ) From ecdeaf78472f80845dbe4a04362e6bc3a3d4aaa1 Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Thu, 4 Feb 2021 12:55:22 -0700 Subject: [PATCH 02/24] install_requires= --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4fe6afcc..19092dee 100644 --- a/setup.py +++ b/setup.py @@ -293,5 +293,5 @@ def run(self): distclass=Distribution, cmdclass=cmdclass, python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', - install_requires: ['requests'], + install_requires=['requests'], ) From e479236a3aab9fcd16a9f760f0b7935a8c22358b Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Thu, 4 Feb 2021 13:00:15 -0700 Subject: [PATCH 03/24] temporarily include ruyaml until can get the parser until pyyaml --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 19092dee..60f933e9 100644 --- a/setup.py +++ b/setup.py @@ -293,5 +293,5 @@ def run(self): distclass=Distribution, cmdclass=cmdclass, python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', - install_requires=['requests'], + install_requires=['ruyaml'], ) From 4669dadf1d77614537aa5918dd03f45d901ac865 Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Thu, 4 Feb 2021 13:18:30 -0700 Subject: [PATCH 04/24] add kwarg to safe_load --- lib3/yaml/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib3/yaml/__init__.py b/lib3/yaml/__init__.py index 86d07b55..66167aa5 100644 --- a/lib3/yaml/__init__.py +++ b/lib3/yaml/__init__.py @@ -17,6 +17,8 @@ import io +import ruyaml + #------------------------------------------------------------------------------ # Warnings control #------------------------------------------------------------------------------ @@ -151,7 +153,7 @@ def full_load_all(stream): """ return load_all(stream, FullLoader) -def safe_load(stream): +def safe_load(stream, yaml_version='1.1'): """ Parse the first YAML document in a stream and produce the corresponding Python object. @@ -159,6 +161,8 @@ def safe_load(stream): Resolve only basic YAML tags. This is known to be safe for untrusted input. """ + if yaml_version == '1.2': + return ruyaml.YAML(typ='safe').load(stream) return load(stream, SafeLoader) def safe_load_all(stream): From a7b82df1e28eb1540308a45e424aade0e515e25c Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Thu, 4 Feb 2021 13:21:13 -0700 Subject: [PATCH 05/24] Drop Python 2.7 support. --- .github/workflows/ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c67847b2..2b59d7b8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -121,7 +121,6 @@ jobs: python_tag: # NB manylinux >=2014 containers don't have Python 2.7, so we have to # use exclude to skip it. - - cp27-cp27mu - cp36-cp36m - cp37-cp37m - cp38-cp38 From b02ee41e695dab987c1ec3956768b9367b8774e1 Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Thu, 4 Feb 2021 13:24:10 -0700 Subject: [PATCH 06/24] remove the Mac test for 2.7 as well --- .github/workflows/ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2b59d7b8..c933385d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -211,7 +211,6 @@ jobs: arch: - x86_64 python_tag: - - cp27* - cp36* - cp37* - cp38* From 320799a8ddbadc9a8b19f578dc7c3c4bd60545e8 Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Thu, 4 Feb 2021 13:34:24 -0700 Subject: [PATCH 07/24] setup() imports test_all imports yaml so no dependencies can be installed until after setup() --- lib3/yaml/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib3/yaml/__init__.py b/lib3/yaml/__init__.py index 66167aa5..b97a5364 100644 --- a/lib3/yaml/__init__.py +++ b/lib3/yaml/__init__.py @@ -17,8 +17,6 @@ import io -import ruyaml - #------------------------------------------------------------------------------ # Warnings control #------------------------------------------------------------------------------ @@ -162,6 +160,7 @@ def safe_load(stream, yaml_version='1.1'): to be safe for untrusted input. """ if yaml_version == '1.2': + import ruyaml return ruyaml.YAML(typ='safe').load(stream) return load(stream, SafeLoader) From cdd1c3a860981a45b57ded0119c6ba7e853d28c0 Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Thu, 4 Feb 2021 14:18:43 -0700 Subject: [PATCH 08/24] Remove py27. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8694f6db..aad7b092 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,pypy,py35,py36,py37,py38,py39 +envlist = pypy,py35,py36,py37,py38,py39 [testenv] deps = From 992ea3cf21ca0e844be6b8e41802863b3300ab43 Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Fri, 5 Feb 2021 09:56:45 -0700 Subject: [PATCH 09/24] test_recursive_12 --- lib3/yaml/dumper.py | 4 +++- lib3/yaml/loader.py | 4 +++- setup.py | 2 +- tests/lib3/test_recursive.py | 27 +++++++++++++++++++++++++++ tox.ini | 1 + 5 files changed, 35 insertions(+), 3 deletions(-) diff --git a/lib3/yaml/dumper.py b/lib3/yaml/dumper.py index 6aadba55..c6efdc14 100644 --- a/lib3/yaml/dumper.py +++ b/lib3/yaml/dumper.py @@ -1,11 +1,13 @@ -__all__ = ['BaseDumper', 'SafeDumper', 'Dumper'] +__all__ = ['BaseDumper', 'SafeDumper', 'Dumper', 'YAML12Dumper'] from .emitter import * from .serializer import * from .representer import * from .resolver import * +from ruyaml import Dumper as YAML12Dumper + class BaseDumper(Emitter, Serializer, BaseRepresenter, BaseResolver): def __init__(self, stream, diff --git a/lib3/yaml/loader.py b/lib3/yaml/loader.py index e90c1122..572df52d 100644 --- a/lib3/yaml/loader.py +++ b/lib3/yaml/loader.py @@ -1,5 +1,5 @@ -__all__ = ['BaseLoader', 'FullLoader', 'SafeLoader', 'Loader', 'UnsafeLoader'] +__all__ = ['BaseLoader', 'FullLoader', 'SafeLoader', 'Loader', 'UnsafeLoader', 'YAML12UnsafeLoader'] from .reader import * from .scanner import * @@ -8,6 +8,8 @@ from .constructor import * from .resolver import * +from ruyaml import Loader as YAML12UnsafeLoader + class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, BaseResolver): def __init__(self, stream): diff --git a/setup.py b/setup.py index 60f933e9..1f626a3a 100644 --- a/setup.py +++ b/setup.py @@ -293,5 +293,5 @@ def run(self): distclass=Distribution, cmdclass=cmdclass, python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*', - install_requires=['ruyaml'], + install_requires=['ruyaml @ git+https://github.com/dHannasch/ruyaml.git@sort-keys'], ) diff --git a/tests/lib3/test_recursive.py b/tests/lib3/test_recursive.py index 08042c81..36fe0979 100644 --- a/tests/lib3/test_recursive.py +++ b/tests/lib3/test_recursive.py @@ -1,6 +1,8 @@ import yaml +# We could use something like @loaderdumper.register_class +# instead of the unsafe loader. class AnInstance: def __init__(self, foo, bar): @@ -45,6 +47,31 @@ def test_recursive(recursive_filename, verbose=False): test_recursive.unittest = ['.recursive'] +def test_recursive_12(recursive_filename, verbose=False): + context = globals().copy() + exec(open(recursive_filename, 'rb').read(), context) + value1 = context['value'] + output1 = None + value2 = None + output2 = None + # ruyaml.register_class(AnInstanceWithState) + try: + output1 = yaml.dump(value1, Dumper=yaml.YAML12Dumper, sort_keys=False) + value2 = yaml.load(output1, Loader=yaml.YAML12UnsafeLoader) + output2 = yaml.dump(value2, Dumper=yaml.YAML12Dumper, sort_keys=False) + # raise Exception(value1, output1, value2, output2) + assert output1 == output2, (output1, output2) + finally: + if verbose: + print("VALUE1:", value1) + print("VALUE2:", value2) + print("OUTPUT1:") + print(output1) + print("OUTPUT2:") + print(output2) + +test_recursive_12.unittest = ['.recursive'] + if __name__ == '__main__': import test_appliance test_appliance.run(globals()) diff --git a/tox.ini b/tox.ini index 8694f6db..8d26e53f 100644 --- a/tox.ini +++ b/tox.ini @@ -4,6 +4,7 @@ envlist = py27,pypy,py35,py36,py37,py38,py39 [testenv] deps = Cython + ruyaml commands = python setup.py test passenv = PYYAML_TEST_GROUP From 6c7966a17a952bc21dc452984bfe9d4f64b61226 Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Fri, 5 Feb 2021 10:10:36 -0700 Subject: [PATCH 10/24] Install before test. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 69efbdc7..18fd9773 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ test: build testext: buildext ${PYTHON} tests/lib/test_build_ext.py ${TEST} -testall: +testall: install ${PYTHON} setup.py test dist: From 7ba163be3f64ba96343959a9c4a9b6ed803a6f08 Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Fri, 5 Feb 2021 10:27:31 -0700 Subject: [PATCH 11/24] Use pip in the Makefile, like the packaging/build scripts already do. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 69efbdc7..c5da35d2 100644 --- a/Makefile +++ b/Makefile @@ -18,10 +18,10 @@ forceext: ${PYTHON} setup.py --with-libyaml build -f ${PARAMETERS} install: - ${PYTHON} setup.py install ${PARAMETERS} + ${PYTHON} -m pip install --editable . ${PARAMETERS} installext: - ${PYTHON} setup.py --with-libyaml install ${PARAMETERS} + ${PYTHON} -m pip install --editable . --install-option="--with-libyaml" ${PARAMETERS} test: build ${PYTHON} tests/lib/test_build.py ${TEST} From c01b0b103a1a6bec8115c6806bb32af3f86bea4c Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Fri, 5 Feb 2021 10:47:53 -0700 Subject: [PATCH 12/24] Drop Python 2.7 from Windows. --- packaging/build/appveyor.ps1 | 2 -- 1 file changed, 2 deletions(-) diff --git a/packaging/build/appveyor.ps1 b/packaging/build/appveyor.ps1 index d4cc6cf7..140f36f6 100644 --- a/packaging/build/appveyor.ps1 +++ b/packaging/build/appveyor.ps1 @@ -117,8 +117,6 @@ Function Upload-Artifacts() { Bootstrap $pythons = @( -"C:\Python27" -"C:\Python27-x64" "C:\Python36" "C:\Python36-x64" "C:\Python37" From 60b27c496472f530210eba9a9b8aad78fc72d412 Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Fri, 5 Feb 2021 11:24:47 -0700 Subject: [PATCH 13/24] Use pip on AppVeyor as well. --- packaging/build/appveyor.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/build/appveyor.ps1 b/packaging/build/appveyor.ps1 index d4cc6cf7..92b388bb 100644 --- a/packaging/build/appveyor.ps1 +++ b/packaging/build/appveyor.ps1 @@ -103,7 +103,7 @@ Function Build-Wheel($python_path) { Invoke-Exe { cmake.exe --build . --config Release } popd - Invoke-Exe { & $python setup.py --with-libyaml build_ext -I libyaml\include -L libyaml\build\Release -D YAML_DECLARE_STATIC build test bdist_wheel } + Invoke-Exe { & $python -m pip install --editable . --install-option="--with-libyaml build_ext -I libyaml\include -L libyaml\build\Release -D YAML_DECLARE_STATIC build test bdist_wheel" } } Function Upload-Artifacts() { From fb1580c00e230dffa96f01f7cc2d9fa7fd5b37bd Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Fri, 5 Feb 2021 11:39:09 -0700 Subject: [PATCH 14/24] need to do this in two steps --- packaging/build/appveyor.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packaging/build/appveyor.ps1 b/packaging/build/appveyor.ps1 index 244ce7e4..2cba4804 100644 --- a/packaging/build/appveyor.ps1 +++ b/packaging/build/appveyor.ps1 @@ -103,7 +103,8 @@ Function Build-Wheel($python_path) { Invoke-Exe { cmake.exe --build . --config Release } popd - Invoke-Exe { & $python -m pip install --editable . --install-option="--with-libyaml build_ext -I libyaml\include -L libyaml\build\Release -D YAML_DECLARE_STATIC build test bdist_wheel" } + Invoke-Exe { & $python -m pip install --editable . } + Invoke-Exe { & $python setup.py --with-libyaml build_ext -I libyaml\include -L libyaml\build\Release -D YAML_DECLARE_STATIC build test bdist_wheel } } Function Upload-Artifacts() { From 76cbcb9b521548248e309fb2781e61a41f28abaf Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Fri, 5 Feb 2021 12:02:06 -0700 Subject: [PATCH 15/24] --with-libyaml probably irrelevant with --editable --- packaging/build/appveyor.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/build/appveyor.ps1 b/packaging/build/appveyor.ps1 index 2cba4804..e6df4e08 100644 --- a/packaging/build/appveyor.ps1 +++ b/packaging/build/appveyor.ps1 @@ -103,7 +103,7 @@ Function Build-Wheel($python_path) { Invoke-Exe { cmake.exe --build . --config Release } popd - Invoke-Exe { & $python -m pip install --editable . } + Invoke-Exe { & $python -m pip install --editable . --install-option="--with-libyaml" } Invoke-Exe { & $python setup.py --with-libyaml build_ext -I libyaml\include -L libyaml\build\Release -D YAML_DECLARE_STATIC build test bdist_wheel } } From 60675c021e0ede4fded0be7b09b776d9e0d966a8 Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Fri, 5 Feb 2021 12:10:31 -0700 Subject: [PATCH 16/24] build before test --- packaging/build/appveyor.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packaging/build/appveyor.ps1 b/packaging/build/appveyor.ps1 index e6df4e08..6c3154e2 100644 --- a/packaging/build/appveyor.ps1 +++ b/packaging/build/appveyor.ps1 @@ -103,8 +103,10 @@ Function Build-Wheel($python_path) { Invoke-Exe { cmake.exe --build . --config Release } popd + Invoke-Exe { & $python -m pip install --editable . } + Invoke-Exe { & $python setup.py --with-libyaml build_ext -I libyaml\include -L libyaml\build\Release -D YAML_DECLARE_STATIC build } Invoke-Exe { & $python -m pip install --editable . --install-option="--with-libyaml" } - Invoke-Exe { & $python setup.py --with-libyaml build_ext -I libyaml\include -L libyaml\build\Release -D YAML_DECLARE_STATIC build test bdist_wheel } + Invoke-Exe { & $python setup.py --with-libyaml test bdist_wheel } } Function Upload-Artifacts() { From 3eec227c7201ee6ae105fdd58efe79125da89375 Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Fri, 5 Feb 2021 12:22:18 -0700 Subject: [PATCH 17/24] check libyaml after reinstall --- packaging/build/appveyor.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packaging/build/appveyor.ps1 b/packaging/build/appveyor.ps1 index 6c3154e2..780fc6a4 100644 --- a/packaging/build/appveyor.ps1 +++ b/packaging/build/appveyor.ps1 @@ -104,8 +104,10 @@ Function Build-Wheel($python_path) { popd Invoke-Exe { & $python -m pip install --editable . } + Invoke-Exe { & $python -m pip uninstall --yes pyyaml } Invoke-Exe { & $python setup.py --with-libyaml build_ext -I libyaml\include -L libyaml\build\Release -D YAML_DECLARE_STATIC build } - Invoke-Exe { & $python -m pip install --editable . --install-option="--with-libyaml" } + Invoke-Exe { & $python -m pip install --editable . } + Invoke-Exe { & $python -c "import yaml; assert yaml.__with_libyaml__" } Invoke-Exe { & $python setup.py --with-libyaml test bdist_wheel } } From 2b36a0e08f56ed4730c23271b9db36765f994298 Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Fri, 5 Feb 2021 12:45:45 -0700 Subject: [PATCH 18/24] setup.py test does manual path-hacking to get the build directory --- packaging/build/appveyor.ps1 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packaging/build/appveyor.ps1 b/packaging/build/appveyor.ps1 index 780fc6a4..98a5130f 100644 --- a/packaging/build/appveyor.ps1 +++ b/packaging/build/appveyor.ps1 @@ -106,8 +106,7 @@ Function Build-Wheel($python_path) { Invoke-Exe { & $python -m pip install --editable . } Invoke-Exe { & $python -m pip uninstall --yes pyyaml } Invoke-Exe { & $python setup.py --with-libyaml build_ext -I libyaml\include -L libyaml\build\Release -D YAML_DECLARE_STATIC build } - Invoke-Exe { & $python -m pip install --editable . } - Invoke-Exe { & $python -c "import yaml; assert yaml.__with_libyaml__" } + # setup.py test does manual path-hacking to get the build directory Invoke-Exe { & $python setup.py --with-libyaml test bdist_wheel } } From 22f6fbe89405a231fb3d47324e30cc6baf29ba58 Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Fri, 5 Feb 2021 14:30:56 -0700 Subject: [PATCH 19/24] Add sanity-check tests that are installed with the library and can be run after installation. --- lib3/yaml/tests/__init__.py | 0 lib3/yaml/tests/roundtrip.py | 77 ++++++++++++++++++++ lib3/yaml/tests/test_add_xxx.py | 120 ++++++++++++++++++++++++++++++++ 3 files changed, 197 insertions(+) create mode 100644 lib3/yaml/tests/__init__.py create mode 100644 lib3/yaml/tests/roundtrip.py create mode 100644 lib3/yaml/tests/test_add_xxx.py diff --git a/lib3/yaml/tests/__init__.py b/lib3/yaml/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/lib3/yaml/tests/roundtrip.py b/lib3/yaml/tests/roundtrip.py new file mode 100644 index 00000000..654ce805 --- /dev/null +++ b/lib3/yaml/tests/roundtrip.py @@ -0,0 +1,77 @@ +# coding: utf-8 + +from __future__ import print_function + +""" +helper routines for testing round trip of commented YAML data +""" +import sys +import textwrap +from pathlib import Path + +enforce = object() + + +def dedent(data): + try: + position_of_first_newline = data.index('\n') + for idx in range(position_of_first_newline): + if not data[idx].isspace(): + raise ValueError + except ValueError: + pass + else: + data = data[position_of_first_newline + 1 :] + return textwrap.dedent(data) + + +def diff(inp, outp, file_name='stdin'): + import difflib + + inl = inp.splitlines(True) # True for keepends + outl = outp.splitlines(True) + diff = difflib.unified_diff(inl, outl, file_name, 'round trip YAML') + # 2.6 difflib has trailing space on filename lines %-) + strip_trailing_space = sys.version_info < (2, 7) + for line in diff: + if strip_trailing_space and line[:4] in ['--- ', '+++ ']: + line = line.rstrip() + '\n' + sys.stdout.write(line) + + + + +def save_and_run(program, base_dir=None, output=None, file_name=None, optimized=False): + """ + safe and run a python program, thereby circumventing any restrictions on module level + imports + """ + from subprocess import STDOUT, CalledProcessError, check_output + + if not hasattr(base_dir, 'hash'): + base_dir = Path(str(base_dir)) + if file_name is None: + file_name = 'safe_and_run_tmp.py' + file_name = base_dir / file_name + file_name.write_text(dedent(program)) + + try: + cmd = [sys.executable] + if optimized: + cmd.append('-O') + cmd.append(str(file_name)) + print('running:', *cmd) + res = check_output(cmd, stderr=STDOUT, universal_newlines=True) + if output is not None: + if '__pypy__' in sys.builtin_module_names: + res = res.splitlines(True) + res = [line for line in res if 'no version info' not in line] + res = ''.join(res) + print('result: ', res, end='') + print('expected:', output, end='') + assert res == output + except CalledProcessError as exception: + print("##### Running '{} {}' FAILED #####".format(sys.executable, file_name)) + print(exception.output) + return exception.returncode + return 0 diff --git a/lib3/yaml/tests/test_add_xxx.py b/lib3/yaml/tests/test_add_xxx.py new file mode 100644 index 00000000..a9d317db --- /dev/null +++ b/lib3/yaml/tests/test_add_xxx.py @@ -0,0 +1,120 @@ +# coding: utf-8 + +import re + +import pytest # NOQA +from .roundtrip import dedent + + +# from PyYAML docs +class Dice(tuple): + def __new__(cls, a, b): + return tuple.__new__(cls, [a, b]) + + def __repr__(self): + return 'Dice(%s,%s)' % self + + +def dice_constructor(loader, node): + value = loader.construct_scalar(node) + a, b = map(int, value.split('d')) + return Dice(a, b) + + +def dice_representer(dumper, data): + return dumper.represent_scalar(u'!dice', u'{}d{}'.format(*data)) + + +def test_dice_constructor(): + import yaml # NOQA + + yaml.add_constructor(u'!dice', dice_constructor) + data = yaml.load('initial hit points: !dice 8d4', Loader=yaml.Loader) + assert str(data) == "{'initial hit points': Dice(8,4)}" + + +def test_dice_constructor_with_loader(): + import yaml # NOQA + + yaml.add_constructor(u'!dice', dice_constructor, Loader=yaml.Loader) + data = yaml.load('initial hit points: !dice 8d4', Loader=yaml.Loader) + assert str(data) == "{'initial hit points': Dice(8,4)}" + + +def test_dice_representer(): + import yaml # NOQA + + yaml.add_representer(Dice, dice_representer) + assert ( + yaml.dump(dict(gold=Dice(10, 6)), default_flow_style=False) + == 'gold: !dice 10d6\n' + ) + + +def test_dice_implicit_resolver(): + import yaml # NOQA + + pattern = re.compile(r'^\d+d\d+$') + yaml.add_implicit_resolver(u'!dice', pattern) + assert ( + yaml.dump(dict(treasure=Dice(10, 20)), default_flow_style=False) + == 'treasure: 10d20\n' + ) + assert yaml.load('damage: 5d10', Loader=yaml.Loader) == dict(damage=Dice(5, 10)) + + +class Obj1(dict): + def __init__(self, suffix): + self._suffix = suffix + self._node = None + + def add_node(self, n): + self._node = n + + def __repr__(self): + return 'Obj1(%s->%s)' % (self._suffix, self.items()) + + def dump(self): + return repr(self._node) + + +class YAMLObj1: + yaml_tag = u'!obj:' + + @classmethod + def from_yaml(cls, loader, suffix, node): + import yaml # NOQA + + obj1 = Obj1(suffix) + if isinstance(node, yaml.MappingNode): + obj1.add_node(loader.construct_mapping(node)) + else: + raise NotImplementedError + return obj1 + + @classmethod + def to_yaml(cls, dumper, data): + return dumper.represent_scalar(cls.yaml_tag + data._suffix, data.dump()) + + +def test_yaml_obj(): + import yaml # NOQA + + yaml.add_representer(Obj1, YAMLObj1.to_yaml) + yaml.add_multi_constructor(YAMLObj1.yaml_tag, YAMLObj1.from_yaml) + x = yaml.load('!obj:x.2\na: 1', Loader=yaml.Loader) + print(x) + assert yaml.dump(x) == """!obj:x.2 "{'a': 1}"\n""" + + +def test_yaml_obj_with_loader_and_dumper(): + import yaml # NOQA + + yaml.add_representer(Obj1, YAMLObj1.to_yaml, Dumper=yaml.Dumper) + yaml.add_multi_constructor( + YAMLObj1.yaml_tag, YAMLObj1.from_yaml, Loader=yaml.Loader + ) + x = yaml.load('!obj:x.2\na: 1', Loader=yaml.Loader) + print(x) + assert yaml.dump(x) == """!obj:x.2 "{'a': 1}"\n""" + From c4438ca36260e8306289befed6dca0691ddad027 Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Fri, 5 Feb 2021 14:33:34 -0700 Subject: [PATCH 20/24] pytest --pyargs yaml --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 18fd9773..e76c19a2 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,7 @@ testext: buildext ${PYTHON} tests/lib/test_build_ext.py ${TEST} testall: install + ${PYTHON} -m pytest --pyargs yaml ${PYTHON} setup.py test dist: From 370946822c82c89b286ac3839c6445091b266812 Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Fri, 5 Feb 2021 14:38:05 -0700 Subject: [PATCH 21/24] pip install pytest --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index e76c19a2..e7b22305 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,7 @@ testext: buildext ${PYTHON} tests/lib/test_build_ext.py ${TEST} testall: install + ${PYTHON} -m pip install pytest ${PYTHON} -m pytest --pyargs yaml ${PYTHON} setup.py test From 6a1a26f1b851ec59723132e6daa5b7dc818585bf Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Fri, 5 Feb 2021 14:48:18 -0700 Subject: [PATCH 22/24] Use YAML 1.2 Loader and Dumper for these tests --- lib3/yaml/tests/test_add_xxx.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib3/yaml/tests/test_add_xxx.py b/lib3/yaml/tests/test_add_xxx.py index a9d317db..c00c9781 100644 --- a/lib3/yaml/tests/test_add_xxx.py +++ b/lib3/yaml/tests/test_add_xxx.py @@ -29,15 +29,15 @@ def test_dice_constructor(): import yaml # NOQA yaml.add_constructor(u'!dice', dice_constructor) - data = yaml.load('initial hit points: !dice 8d4', Loader=yaml.Loader) + data = yaml.load('initial hit points: !dice 8d4', Loader=yaml.YAML12Loader) assert str(data) == "{'initial hit points': Dice(8,4)}" def test_dice_constructor_with_loader(): import yaml # NOQA - yaml.add_constructor(u'!dice', dice_constructor, Loader=yaml.Loader) - data = yaml.load('initial hit points: !dice 8d4', Loader=yaml.Loader) + yaml.add_constructor(u'!dice', dice_constructor, Loader=yaml.YAML12Loader) + data = yaml.load('initial hit points: !dice 8d4', Loader=yaml.YAML12Loader) assert str(data) == "{'initial hit points': Dice(8,4)}" @@ -60,7 +60,7 @@ def test_dice_implicit_resolver(): yaml.dump(dict(treasure=Dice(10, 20)), default_flow_style=False) == 'treasure: 10d20\n' ) - assert yaml.load('damage: 5d10', Loader=yaml.Loader) == dict(damage=Dice(5, 10)) + assert yaml.load('damage: 5d10', Loader=yaml.YAML12Loader) == dict(damage=Dice(5, 10)) class Obj1(dict): @@ -102,7 +102,7 @@ def test_yaml_obj(): yaml.add_representer(Obj1, YAMLObj1.to_yaml) yaml.add_multi_constructor(YAMLObj1.yaml_tag, YAMLObj1.from_yaml) - x = yaml.load('!obj:x.2\na: 1', Loader=yaml.Loader) + x = yaml.load('!obj:x.2\na: 1', Loader=yaml.YAML12Loader) print(x) assert yaml.dump(x) == """!obj:x.2 "{'a': 1}"\n""" @@ -110,11 +110,11 @@ def test_yaml_obj(): def test_yaml_obj_with_loader_and_dumper(): import yaml # NOQA - yaml.add_representer(Obj1, YAMLObj1.to_yaml, Dumper=yaml.Dumper) + yaml.add_representer(Obj1, YAMLObj1.to_yaml, Dumper=yaml.YAML12Dumper) yaml.add_multi_constructor( - YAMLObj1.yaml_tag, YAMLObj1.from_yaml, Loader=yaml.Loader + YAMLObj1.yaml_tag, YAMLObj1.from_yaml, Loader=yaml.YAML12Loader ) - x = yaml.load('!obj:x.2\na: 1', Loader=yaml.Loader) + x = yaml.load('!obj:x.2\na: 1', Loader=yaml.YAML12Loader) print(x) assert yaml.dump(x) == """!obj:x.2 "{'a': 1}"\n""" From 0629ce283393eb25ac8aa05b79484d6f468d09d7 Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Fri, 5 Feb 2021 15:08:50 -0700 Subject: [PATCH 23/24] add YAML12Loader --- lib3/yaml/loader.py | 3 ++- lib3/yaml/tests/test_add_xxx.py | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/lib3/yaml/loader.py b/lib3/yaml/loader.py index 572df52d..de2ce37f 100644 --- a/lib3/yaml/loader.py +++ b/lib3/yaml/loader.py @@ -1,5 +1,5 @@ -__all__ = ['BaseLoader', 'FullLoader', 'SafeLoader', 'Loader', 'UnsafeLoader', 'YAML12UnsafeLoader'] +__all__ = ['BaseLoader', 'FullLoader', 'SafeLoader', 'Loader', 'UnsafeLoader', 'YAML12Loader', 'YAML12UnsafeLoader'] from .reader import * from .scanner import * @@ -9,6 +9,7 @@ from .resolver import * from ruyaml import Loader as YAML12UnsafeLoader +from ruyaml import Loader as YAML12Loader class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, BaseResolver): diff --git a/lib3/yaml/tests/test_add_xxx.py b/lib3/yaml/tests/test_add_xxx.py index c00c9781..16abcb41 100644 --- a/lib3/yaml/tests/test_add_xxx.py +++ b/lib3/yaml/tests/test_add_xxx.py @@ -28,7 +28,7 @@ def dice_representer(dumper, data): def test_dice_constructor(): import yaml # NOQA - yaml.add_constructor(u'!dice', dice_constructor) + yaml.add_constructor(u'!dice', dice_constructor, Loader=yaml.YAML12Loader) data = yaml.load('initial hit points: !dice 8d4', Loader=yaml.YAML12Loader) assert str(data) == "{'initial hit points': Dice(8,4)}" @@ -44,9 +44,9 @@ def test_dice_constructor_with_loader(): def test_dice_representer(): import yaml # NOQA - yaml.add_representer(Dice, dice_representer) + yaml.add_representer(Dice, dice_representer, Dumper=yaml.YAML12Dumper) assert ( - yaml.dump(dict(gold=Dice(10, 6)), default_flow_style=False) + yaml.dump(dict(gold=Dice(10, 6)), default_flow_style=False, sort_keys=False, Dumper=yaml.YAML12Dumper) == 'gold: !dice 10d6\n' ) @@ -55,9 +55,9 @@ def test_dice_implicit_resolver(): import yaml # NOQA pattern = re.compile(r'^\d+d\d+$') - yaml.add_implicit_resolver(u'!dice', pattern) + yaml.add_implicit_resolver(u'!dice', pattern, Dumper=yaml.YAML12Dumper) assert ( - yaml.dump(dict(treasure=Dice(10, 20)), default_flow_style=False) + yaml.dump(dict(treasure=Dice(10, 20)), default_flow_style=False, sort_keys=False, Dumper=yaml.YAML12Dumper) == 'treasure: 10d20\n' ) assert yaml.load('damage: 5d10', Loader=yaml.YAML12Loader) == dict(damage=Dice(5, 10)) @@ -83,10 +83,10 @@ class YAMLObj1: @classmethod def from_yaml(cls, loader, suffix, node): - import yaml # NOQA + import ruyaml # NOQA obj1 = Obj1(suffix) - if isinstance(node, yaml.MappingNode): + if isinstance(node, ruyaml.MappingNode): obj1.add_node(loader.construct_mapping(node)) else: raise NotImplementedError @@ -100,11 +100,11 @@ def to_yaml(cls, dumper, data): def test_yaml_obj(): import yaml # NOQA - yaml.add_representer(Obj1, YAMLObj1.to_yaml) - yaml.add_multi_constructor(YAMLObj1.yaml_tag, YAMLObj1.from_yaml) + yaml.add_representer(Obj1, YAMLObj1.to_yaml, Dumper=yaml.YAML12Dumper) + yaml.add_multi_constructor(YAMLObj1.yaml_tag, YAMLObj1.from_yaml, Loader=yaml.YAML12Loader) x = yaml.load('!obj:x.2\na: 1', Loader=yaml.YAML12Loader) print(x) - assert yaml.dump(x) == """!obj:x.2 "{'a': 1}"\n""" + assert yaml.dump(x, sort_keys=False, Dumper=yaml.YAML12Dumper) == """!obj:x.2 "{'a': 1}"\n""" def test_yaml_obj_with_loader_and_dumper(): @@ -116,5 +116,5 @@ def test_yaml_obj_with_loader_and_dumper(): ) x = yaml.load('!obj:x.2\na: 1', Loader=yaml.YAML12Loader) print(x) - assert yaml.dump(x) == """!obj:x.2 "{'a': 1}"\n""" + assert yaml.dump(x, sort_keys=False, Dumper=yaml.YAML12Dumper) == """!obj:x.2 "{'a': 1}"\n""" From 39742710c7ee996da8a8d521ddc8dc6719694e0c Mon Sep 17 00:00:00 2001 From: "David A. Hannasch" Date: Fri, 5 Feb 2021 15:17:21 -0700 Subject: [PATCH 24/24] just use YAML12SafeLoader --- lib3/yaml/__init__.py | 3 +-- lib3/yaml/loader.py | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib3/yaml/__init__.py b/lib3/yaml/__init__.py index b97a5364..9e387624 100644 --- a/lib3/yaml/__init__.py +++ b/lib3/yaml/__init__.py @@ -160,8 +160,7 @@ def safe_load(stream, yaml_version='1.1'): to be safe for untrusted input. """ if yaml_version == '1.2': - import ruyaml - return ruyaml.YAML(typ='safe').load(stream) + return load(stream, YAML12SafeLoader) return load(stream, SafeLoader) def safe_load_all(stream): diff --git a/lib3/yaml/loader.py b/lib3/yaml/loader.py index de2ce37f..8c6012f4 100644 --- a/lib3/yaml/loader.py +++ b/lib3/yaml/loader.py @@ -1,5 +1,5 @@ -__all__ = ['BaseLoader', 'FullLoader', 'SafeLoader', 'Loader', 'UnsafeLoader', 'YAML12Loader', 'YAML12UnsafeLoader'] +__all__ = ['BaseLoader', 'FullLoader', 'SafeLoader', 'Loader', 'UnsafeLoader', 'YAML12Loader', 'YAML12SafeLoader', 'YAML12UnsafeLoader'] from .reader import * from .scanner import * @@ -10,6 +10,7 @@ from ruyaml import Loader as YAML12UnsafeLoader from ruyaml import Loader as YAML12Loader +from ruyaml import SafeLoader as YAML12SafeLoader class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, BaseResolver):