From d9d3a9db476990e537c3a4eb1f8863b36737cb3b Mon Sep 17 00:00:00 2001 From: kclowes Date: Wed, 5 Oct 2022 14:33:21 -0600 Subject: [PATCH] Update protobuf --- ethpm/_utils/ipfs.py | 5 +- ethpm/_utils/protobuf/ipfs_file_pb2.py | 327 ++----------------------- ethpm/assets/ipfs_file.proto | 32 +++ newsfragments/2666.bugfix.rst | 1 + setup.py | 3 +- tox.ini | 8 +- 6 files changed, 61 insertions(+), 315 deletions(-) create mode 100644 ethpm/assets/ipfs_file.proto create mode 100644 newsfragments/2666.bugfix.rst diff --git a/ethpm/_utils/ipfs.py b/ethpm/_utils/ipfs.py index b838bce475..9d3f9dd177 100644 --- a/ethpm/_utils/ipfs.py +++ b/ethpm/_utils/ipfs.py @@ -19,7 +19,7 @@ Descriptor, ) -from ethpm._utils.protobuf.ipfs_file_pb2 import ( +from ethpm._utils.protobuf.ipfs_file_pb2 import ( # type: ignore Data, PBNode, ) @@ -91,8 +91,7 @@ def serialize_bytes(file_bytes: bytes) -> Descriptor: file_size = len(file_bytes) data_protobuf = Data( - # type ignored b/c DataType is manually attached in ipfs_file_pb2.py - Type=Data.DataType.Value("File"), # type: ignore + Type=Data.DataType.Value("File"), Data=file_bytes, filesize=file_size, ) diff --git a/ethpm/_utils/protobuf/ipfs_file_pb2.py b/ethpm/_utils/protobuf/ipfs_file_pb2.py index 344dc1500b..bf07253ead 100644 --- a/ethpm/_utils/protobuf/ipfs_file_pb2.py +++ b/ethpm/_utils/protobuf/ipfs_file_pb2.py @@ -1,318 +1,31 @@ -# flake8: noqa +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! -# source: file.proto - -import sys - -from google.protobuf import ( - descriptor as _descriptor, - descriptor_pb2, - message as _message, - reflection as _reflection, - symbol_database as _symbol_database, -) - -_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1")) +# source: ipfs_file.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() -DESCRIPTOR = _descriptor.FileDescriptor( - name="file.proto", - package="", - syntax="proto2", - serialized_pb=_b( - '\n\nfile.proto"\xa1\x01\n\x04\x44\x61ta\x12\x1c\n\x04Type\x18\x01 \x02(\x0e\x32\x0e.Data.DataType\x12\x0c\n\x04\x44\x61ta\x18\x02 \x01(\x0c\x12\x10\n\x08\x66ilesize\x18\x03 \x01(\x04\x12\x12\n\nblocksizes\x18\x04 \x03(\x04"G\n\x08\x44\x61taType\x12\x07\n\x03Raw\x10\x00\x12\r\n\tDirectory\x10\x01\x12\x08\n\x04\x46ile\x10\x02\x12\x0c\n\x08Metadata\x10\x03\x12\x0b\n\x07Symlink\x10\x04"3\n\x06PBLink\x12\x0c\n\x04Hash\x18\x01 \x01(\x0c\x12\x0c\n\x04Name\x18\x02 \x01(\t\x12\r\n\x05Tsize\x18\x03 \x01(\x04".\n\x06PBNode\x12\x16\n\x05Links\x18\x02 \x03(\x0b\x32\x07.PBLink\x12\x0c\n\x04\x44\x61ta\x18\x01 \x01(\x0c' - ), -) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - -_DATA_DATATYPE = _descriptor.EnumDescriptor( - name="DataType", - full_name="Data.DataType", - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name="Raw", index=0, number=0, options=None, type=None - ), - _descriptor.EnumValueDescriptor( - name="Directory", index=1, number=1, options=None, type=None - ), - _descriptor.EnumValueDescriptor( - name="File", index=2, number=2, options=None, type=None - ), - _descriptor.EnumValueDescriptor( - name="Metadata", index=3, number=3, options=None, type=None - ), - _descriptor.EnumValueDescriptor( - name="Symlink", index=4, number=4, options=None, type=None - ), - ], - containing_type=None, - options=None, - serialized_start=105, - serialized_end=176, -) -_sym_db.RegisterEnumDescriptor(_DATA_DATATYPE) - - -_DATA = _descriptor.Descriptor( - name="Data", - full_name="Data", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="Type", - full_name="Data.Type", - index=0, - number=1, - type=14, - cpp_type=8, - label=2, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="Data", - full_name="Data.Data", - index=1, - number=2, - type=12, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b(""), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="filesize", - full_name="Data.filesize", - index=2, - number=3, - type=4, - cpp_type=4, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="blocksizes", - full_name="Data.blocksizes", - index=3, - number=4, - type=4, - cpp_type=4, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - ], - extensions=[], - nested_types=[], - enum_types=[_DATA_DATATYPE], - options=None, - is_extendable=False, - syntax="proto2", - extension_ranges=[], - oneofs=[], - serialized_start=15, - serialized_end=176, -) - - -_PBLINK = _descriptor.Descriptor( - name="PBLink", - full_name="PBLink", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="Hash", - full_name="PBLink.Hash", - index=0, - number=1, - type=12, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b(""), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="Name", - full_name="PBLink.Name", - index=1, - number=2, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="Tsize", - full_name="PBLink.Tsize", - index=2, - number=3, - type=4, - cpp_type=4, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto2", - extension_ranges=[], - oneofs=[], - serialized_start=178, - serialized_end=229, -) - - -_PBNODE = _descriptor.Descriptor( - name="PBNode", - full_name="PBNode", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="Links", - full_name="PBNode.Links", - index=0, - number=2, - type=11, - cpp_type=10, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="Data", - full_name="PBNode.Data", - index=1, - number=1, - type=12, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b(""), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto2", - extension_ranges=[], - oneofs=[], - serialized_start=231, - serialized_end=277, -) - -_DATA.fields_by_name["Type"].enum_type = _DATA_DATATYPE -_DATA_DATATYPE.containing_type = _DATA -_PBNODE.fields_by_name["Links"].message_type = _PBLINK -DESCRIPTOR.message_types_by_name["Data"] = _DATA -DESCRIPTOR.message_types_by_name["PBLink"] = _PBLINK -DESCRIPTOR.message_types_by_name["PBNode"] = _PBNODE - -Data = _reflection.GeneratedProtocolMessageType( - "Data", - (_message.Message,), - dict( - DESCRIPTOR=_DATA, - __module__="file_pb2" - # @@protoc_insertion_point(class_scope:Data) - ), -) -_sym_db.RegisterMessage(Data) -PBLink = _reflection.GeneratedProtocolMessageType( - "PBLink", - (_message.Message,), - dict( - DESCRIPTOR=_PBLINK, - __module__="file_pb2" - # @@protoc_insertion_point(class_scope:PBLink) - ), -) -_sym_db.RegisterMessage(PBLink) -PBNode = _reflection.GeneratedProtocolMessageType( - "PBNode", - (_message.Message,), - dict( - DESCRIPTOR=_PBNODE, - __module__="file_pb2" - # @@protoc_insertion_point(class_scope:PBNode) - ), -) -_sym_db.RegisterMessage(PBNode) +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0fipfs_file.proto\"\xc1\x01\n\x04\x44\x61ta\x12\x1c\n\x04Type\x18\x01 \x01(\x0e\x32\x0e.Data.DataType\x12\x11\n\x04\x44\x61ta\x18\x02 \x01(\x0cH\x00\x88\x01\x01\x12\x15\n\x08\x66ilesize\x18\x03 \x01(\x04H\x01\x88\x01\x01\x12\x12\n\nblocksizes\x18\x04 \x03(\x04\"G\n\x08\x44\x61taType\x12\x07\n\x03Raw\x10\x00\x12\r\n\tDirectory\x10\x01\x12\x08\n\x04\x46ile\x10\x02\x12\x0c\n\x08Metadata\x10\x03\x12\x0b\n\x07Symlink\x10\x04\x42\x07\n\x05_DataB\x0b\n\t_filesize\"^\n\x06PBLink\x12\x11\n\x04Hash\x18\x01 \x01(\x0cH\x00\x88\x01\x01\x12\x11\n\x04Name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x12\n\x05Tsize\x18\x03 \x01(\x04H\x02\x88\x01\x01\x42\x07\n\x05_HashB\x07\n\x05_NameB\x08\n\x06_Tsize\"<\n\x06PBNode\x12\x16\n\x05Links\x18\x02 \x03(\x0b\x32\x07.PBLink\x12\x11\n\x04\x44\x61ta\x18\x01 \x01(\x0cH\x00\x88\x01\x01\x42\x07\n\x05_Datab\x06proto3') +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'ipfs_file_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + _DATA._serialized_start=20 + _DATA._serialized_end=213 + _DATA_DATATYPE._serialized_start=120 + _DATA_DATATYPE._serialized_end=191 + _PBLINK._serialized_start=215 + _PBLINK._serialized_end=309 + _PBNODE._serialized_start=311 + _PBNODE._serialized_end=371 # @@protoc_insertion_point(module_scope) diff --git a/ethpm/assets/ipfs_file.proto b/ethpm/assets/ipfs_file.proto new file mode 100644 index 0000000000..c3783ea571 --- /dev/null +++ b/ethpm/assets/ipfs_file.proto @@ -0,0 +1,32 @@ +/* This file will generate ipfs_file_pb2.py when invoked on the command line via: */ +/* $ protoc -I=$SRC_DIR --python_out=$DST_DIR $SRC_DIR/ipfs_file.proto */ +/* in this case: */ +/* $ protoc -I=ethpm/assets --python_out=ethpm/_utils/protobuf ethpm/assets/ipfs_file.proto */ + +syntax = "proto3"; + +message Data { + enum DataType { + Raw = 0; + Directory = 1; + File = 2; + Metadata = 3; + Symlink = 4; + } + + DataType Type = 1; + optional bytes Data = 2; + optional uint64 filesize = 3; + repeated uint64 blocksizes = 4; +} + +message PBLink { + optional bytes Hash = 1; + optional string Name = 2; + optional uint64 Tsize = 3; +} + +message PBNode { + repeated PBLink Links = 2; + optional bytes Data = 1; +} diff --git a/newsfragments/2666.bugfix.rst b/newsfragments/2666.bugfix.rst new file mode 100644 index 0000000000..1cd03abf9a --- /dev/null +++ b/newsfragments/2666.bugfix.rst @@ -0,0 +1 @@ +Protobuf dependency had a DoS-able bug. It was fixed in v4.21.6. See: https://nvd.nist.gov/vuln/detail/CVE-2022-1941 diff --git a/setup.py b/setup.py index a909cc6043..2c39474716 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,7 @@ "bumpversion", "flaky>=3.7.0,<4", "hypothesis>=3.31.2,<6", + "importlib-metadata<5.0;python_version<'3.8'", "pytest>=6.2.5,<7", "pytest-asyncio>=0.18.1,<0.19", "pytest-mock>=1.10,<2", @@ -85,7 +86,7 @@ "ipfshttpclient==0.8.0a2", "jsonschema>=4.0.0,<5", "lru-dict>=1.1.6,<2.0.0", - "protobuf==3.20.1", + "protobuf>=4.21.6", "pywin32>=223;platform_system=='Windows'", "requests>=2.16.0,<3.0.0", # remove typing_extensions after python_requires>=3.8, see web3._utils.compat diff --git a/tox.ini b/tox.ini index 63428676e0..c49ad89d8c 100644 --- a/tox.ini +++ b/tox.ini @@ -63,10 +63,10 @@ basepython = basepython=python extras=linter commands= - flake8 {toxinidir}/web3 {toxinidir}/ens {toxinidir}/ethpm {toxinidir}/tests --exclude {toxinidir}/ethpm/ethpm-spec - black {toxinidir}/ens {toxinidir}/ethpm {toxinidir}/web3 {toxinidir}/tests {toxinidir}/setup.py --exclude {toxinidir}/ethpm/ethpm-spec --check - isort --recursive --check-only --diff {toxinidir}/web3/ {toxinidir}/ens/ {toxinidir}/ethpm/ {toxinidir}/tests/ - mypy -p web3 -p ethpm -p ens --config-file {toxinidir}/mypy.ini + flake8 {toxinidir}/web3 {toxinidir}/ens {toxinidir}/ethpm {toxinidir}/tests --exclude {toxinidir}/ethpm/ethpm-spec,{toxinidir}/**/*_pb2.py + black {toxinidir}/ens {toxinidir}/ethpm {toxinidir}/web3 {toxinidir}/tests {toxinidir}/setup.py --exclude {toxinidir}/ethpm/ethpm-spec --extend-exclude {toxinidir}/ethpm/_utils/protobuf/ipfs_file_pb2.py --check + isort --recursive --skip {toxinidir}/ethpm/_utils/protobuf/ipfs_file_pb2.py --skip {toxinidir}/ethpm/ethpm-spec --check-only --diff {toxinidir}/web3/ {toxinidir}/ens/ {toxinidir}/ethpm/ {toxinidir}/tests/ + mypy -p web3 -p ethpm -p ens --exclude {toxinidir}/ethpm/_utils/protobuf/ipfs_file_pb2.py --config-file {toxinidir}/mypy.ini [testenv:benchmark] basepython=python