From 176044be59d6e04611a6375b4c5d5d725f93a96f Mon Sep 17 00:00:00 2001 From: Guillermo Rodriguez Date: Thu, 5 Sep 2024 13:19:37 -0300 Subject: [PATCH] More deserialization fixes & code structure --- poetry.lock | 64 ++++++++++++++++++- pyproject.toml | 1 + src/leap/protocol/abi.py | 8 ++- src/leap/protocol/ds.py | 4 +- src/leap/ship.py | 130 ++++++++++++++++++++------------------- 5 files changed, 142 insertions(+), 65 deletions(-) diff --git a/poetry.lock b/poetry.lock index 6ede621..119c95a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1250,6 +1250,68 @@ files = [ [package.dependencies] h11 = ">=0.9.0,<1" +[[package]] +name = "yappi" +version = "1.6.0" +description = "Yet Another Python Profiler" +optional = false +python-versions = ">=3.6" +files = [ + {file = "yappi-1.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c7c22bb39d3f2f294766f4940848d11b8ad1c43f9cf0a594ed695b07016007fc"}, + {file = "yappi-1.6.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:084688828c0a8c181ebe59bbf15dcd5a5db2f689eada59d5c277b997c4dccf43"}, + {file = "yappi-1.6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23c668e9ce87d70b126f73970cff997a2ab1964b947859ee50580af23964a096"}, + {file = "yappi-1.6.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:d4c7c5de3ae439c53c6c6c98d30d4b063c6fc353428ba3d81b57d91f1c41f654"}, + {file = "yappi-1.6.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:830ffb677b7a9f9886bc7789e9cb75d5b3ad9af5f43d56d48e56431f92f04bcc"}, + {file = "yappi-1.6.0-cp310-cp310-win32.whl", hash = "sha256:b42f7934fe96fd330488f9def51dd8e2fecc5cc9a71dceab8a27a41406b31332"}, + {file = "yappi-1.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:1cd7453e99ebf56491254b0f4c28ae95c5e0ce55043eb17d1ab02c974cbd7416"}, + {file = "yappi-1.6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e9bc33b8ec9bce8b2575a4c3878b3cd223d08eb728669924699e5ac937e7b515"}, + {file = "yappi-1.6.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a1cb70d46827a137350fb84b8fddecd7acec0a11834c763209875788b738f873"}, + {file = "yappi-1.6.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2cefe387bc747afcf0b26c9548e242113e17fac3de2674d900e97eb58a328f6"}, + {file = "yappi-1.6.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:acfbf4c80b6ee0513ad35a6e4a1f633aa2f93357517f9701aed6ad8cd56544d4"}, + {file = "yappi-1.6.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e4959c1dcfb6da8441d05915bfbb9c697e9f11655568f65b87c341e543bd65d5"}, + {file = "yappi-1.6.0-cp311-cp311-win32.whl", hash = "sha256:88dee431bba79866692f444110695133181efb2a6969ab63752f4424787f79c8"}, + {file = "yappi-1.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:8ddbe1475964f145b028f8bf120a58903d8f6c7bdd1be0a16c1471ba2d8646ca"}, + {file = "yappi-1.6.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:1ba7d12c18bc0d092463ad126a95a1b2b8c261c47b0e3bd4cb2fd7479469141c"}, + {file = "yappi-1.6.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fb5bbb4c6b996736554cb8f41e7fb6d5ee6096b7c4f54112cce8cf953a92c0a4"}, + {file = "yappi-1.6.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c12da5f310d81779056566259fef644a9c14ac1ec9a2b1b8a3fc62beb4ca6980"}, + {file = "yappi-1.6.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:677d992c41b239441eee399ac39ea7601010ddb5acb92bf997de7589f9ee2cc1"}, + {file = "yappi-1.6.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:d58e60aac43041d109f0931917204ef02ac01004b9579fe173f2847fbc69655b"}, + {file = "yappi-1.6.0-cp312-cp312-win32.whl", hash = "sha256:a6797f189b7b89154d6c7c53ac769a22f0adb7bd88ea5b8f6c65106a286afad6"}, + {file = "yappi-1.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:cdaa263ba667aac9bf7bdc0d96fd10e2761a287f01fe87dc136f064ab7696af3"}, + {file = "yappi-1.6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:819c43539f55a9f40118ab8b3ce7cb743d66f3af63c7ce984c114533f750b263"}, + {file = "yappi-1.6.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1a8be1f55875e2120d7cddcb7e98c77a79ed87715d6292874c782fcd7da2c50"}, + {file = "yappi-1.6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0f82e4895d04d6fd7bed2bea0abcc84271bdd990371cb053132753b6f5afb73"}, + {file = "yappi-1.6.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1fd7d2da3e32f3d029220356e9b5b24754a7cd708d6e3830405e3dc04ec74153"}, + {file = "yappi-1.6.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:2c7c2c9048b2f9fbd5da9cc65bdad73571023a30b5c34f62d97d9a7d47cbe9f5"}, + {file = "yappi-1.6.0-cp36-cp36m-win32.whl", hash = "sha256:d9f6ac7cd8797850bb6fb4cef8364ed34051be6f47d7da74be3a9261eef4bbfb"}, + {file = "yappi-1.6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6a16dd3fb61cfa7ffeb287312ddfe2a5c61a53693b009d3a7a3f1b8affb9568a"}, + {file = "yappi-1.6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:638bd2458a6dfaa278e8977a8fdf44c8626003c12d8c94d82338ef8aa6ac245f"}, + {file = "yappi-1.6.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:30d294a88baffc3cb13a66fe408ecf8973c927fb3498f327df5af7cc657cdc80"}, + {file = "yappi-1.6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d49f972a8901972b104a2b65953ae8cbe005d5c09e0974422195bb780b2c5001"}, + {file = "yappi-1.6.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:e40a872d146ddddae337181f8808aa6c3b37ffa66bd38a18f009b9e2f2c08b99"}, + {file = "yappi-1.6.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:31050972159a026876a06b5eec97f2dbaaaa291ebf3cf07a0d5506cce37ef339"}, + {file = "yappi-1.6.0-cp37-cp37m-win32.whl", hash = "sha256:36278de1ecf3a781322fb5f9511abc0b66bff327ca87a9e868dc2e376ad1b11a"}, + {file = "yappi-1.6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ef61f5fed7c19dddad5b7565df5b7bdfa861d51c15b01a90d283c4d3c97c42e2"}, + {file = "yappi-1.6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e33c097402e101a51f9963654108e7625853ddc979b562e8381f761cce99ae13"}, + {file = "yappi-1.6.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9ea8dada24af41c11059f70741d8fce707aaf6862a9306f2ab63bde35f0ce9e"}, + {file = "yappi-1.6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e37817722a052632de21674ec8acf59645c08df786920362879a76493037c99e"}, + {file = "yappi-1.6.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:72aff63e74b87ffbf121211246922df9ac5dffc8deabdc6964f5b7f399799d0a"}, + {file = "yappi-1.6.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8a3c970f43f6d9bbc7159b535fbef492cb21576c094e77a673362ad827d9e00a"}, + {file = "yappi-1.6.0-cp38-cp38-win32.whl", hash = "sha256:69a4aced8c86bcc91fbecc3924ca9bd0a91ed715531c8a039199ef325ebb7046"}, + {file = "yappi-1.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:93352217bf560bea09b1cb43a17361bd2d41864698fa7ae46ce1066266c1af76"}, + {file = "yappi-1.6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6678f046e6bffc68ef2d7781f9fc90b932ca6e90ea966371318ed904c4c38b8d"}, + {file = "yappi-1.6.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b2f8f0877e3b85b6d2074d2fb541085cd519481f3df9c7e7011fc3867c364c7e"}, + {file = "yappi-1.6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ae77cfe71682beec6f15ddd6dfb5912436c489b38eb2c3355f4481c07b9c4bf"}, + {file = "yappi-1.6.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:237b1ac310ef364db6d2a1817de93a346d1ed98abfa3053810dbbdcab9ca9300"}, + {file = "yappi-1.6.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:474e9f3ba9394c19dd2f7dc257123e3918c178638597d507ee2094f19d938a39"}, + {file = "yappi-1.6.0-cp39-cp39-win32.whl", hash = "sha256:a69ce9d7acc71419092f158ab4851d982e90ecbffbe6abf7d95516f3f741b57f"}, + {file = "yappi-1.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:e6ec9eabc974f9df868faa67461a9d167e9612083f59338f755f4aa61a2552b7"}, + {file = "yappi-1.6.0.tar.gz", hash = "sha256:a9aaf72009d8c03067294151ee0470ac7a6dfa7b33baab40b198d6c1ef00430a"}, +] + +[package.extras] +test = ["gevent (>=20.6.2)"] + [[package]] name = "zipp" version = "3.20.0" @@ -1326,4 +1388,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = ">=3.9" -content-hash = "4cdf2916f4ad2f618d1ba4a572fb8c8a373031a020ed553d3e7d95f81b2da330" +content-hash = "aef11a7be0081b9ab9a3176a8c42f73c5de63f66c969db0d68d3dee75fffcc6b" diff --git a/pyproject.toml b/pyproject.toml index c998aca..e14e1fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ pdbpp = {version = '^0.10.3'} pytest = {version = '^7.4.2'} docker = {version = '^6.1.3'} pytest-trio = {version = "^0.8.0"} +yappi = "^1.6.0" [tool.poetry.group.snaps] optional = true diff --git a/src/leap/protocol/abi.py b/src/leap/protocol/abi.py index 92c5d96..d4a6d51 100644 --- a/src/leap/protocol/abi.py +++ b/src/leap/protocol/abi.py @@ -278,7 +278,13 @@ def unpack_type(self, type_name: str): if desc.variant: i = self.unpack_uint8() - desc.struct = self.resolve_struct(desc.variant.types[i]) + sub_name = self.resolve_alias(desc.variant.types[i]) + + if sub_name not in STANDARD_TYPES: + desc.struct = self.resolve_struct(sub_name) + + else: + desc.strip_name = sub_name # find right unpacking function unpack_partial = None diff --git a/src/leap/protocol/ds.py b/src/leap/protocol/ds.py index 4ac5598..195cb1d 100644 --- a/src/leap/protocol/ds.py +++ b/src/leap/protocol/ds.py @@ -372,7 +372,9 @@ def unpack_signature(self): data = data + ripmed160(data + b"K1")[:4] return "SIG_K1_" + b58encode(data).decode("ascii") elif t == 1: - raise Exception("not implemented") + data = self.read(65) + data = data + ripmed160(data + b"R1")[:4] + return "SIG_R1_" + b58encode(data).decode("ascii") else: raise Exception("invalid binary signature") diff --git a/src/leap/ship.py b/src/leap/ship.py index 58a784b..3b71e8f 100644 --- a/src/leap/ship.py +++ b/src/leap/ship.py @@ -31,6 +31,68 @@ def decode_block_result( decoded_deltas = [] decoded_actions = [] + def maybe_decode_account_row(row): + if not row['present']: + return + + account_delta = abi_unpack('account', row['data']) + abi_raw = account_delta['abi'] + if len(abi_raw) > 0: + contracts[account_delta['name']] = abi_unpack( + 'abi', abi_raw, spec=ABI) + + def maybe_decode_contract_row(row): + contract_delta = abi_unpack('contract_row', row['data']) + table = contract_delta['table'] + code = contract_delta['code'] + + relevant_tables = delta_whitelist.get(code, []) + if table not in relevant_tables: + return + + contract_abi = contracts.get(code, None) + if not contract_abi: + raise SerializationException(f'abi for contract {code} not found') + + try: + decoded_deltas.append( + abi_unpack( + table, + contract_delta['value'], + abi=contract_abi + ) + ) + + except Exception as e: + e.add_note(f'while decoding table delta {code}::{contract_delta["table"]}') + raise e + + def maybe_decode_tx_trace(tx_trace): + for act_trace in tx_trace['action_traces']: + action = act_trace['act'] + account = action['account'] + name = action['name'] + + if account == 'eosio' and name == 'onblock': + continue + + try: + relevant_actions = action_whitelist.get(account, []) + if name not in relevant_actions: + continue + + contract_abi = contracts.get(account, None) + if not contract_abi: + raise SerializationException(f'abi for contract {account} not found') + + act_data = abi_unpack(name, action['data'], abi=contract_abi) + action['data'] = act_data + decoded_actions.append(action) + + except Exception as e: + e.add_note(f'while decoding action trace {account}::{name}') + raise e + try: if result['block']: block = abi_unpack('signed_block', result['block']) @@ -43,77 +105,21 @@ def decode_block_result( rows = delta['rows'] match delta['name']: case 'account': - for row in rows: - if row['present']: - account_delta = abi_unpack('account', row['data']) - abi_raw = account_delta['abi'] - if len(abi_raw) > 0: - contracts[account_delta['name']] = abi_unpack( - 'abi', abi_raw, spec=ABI) - + [maybe_decode_account_row(row) for row in rows] break case 'contract_row': if len(delta_whitelist) == 0: continue - for row in rows: - contract_delta = abi_unpack('contract_row', row['data']) - table = contract_delta['table'] - code = contract_delta['code'] - - relevant_tables = delta_whitelist.get(code, []) - if table not in relevant_tables: - continue - - contract_abi = contracts.get(code, None) - if not contract_abi: - raise SerializationException(f'abi for contract {code} not found') - - try: - decoded_deltas.append( - abi_unpack( - table, - contract_delta['value'], - abi=contract_abi - ) - ) - - except Exception as e: - e.add_note(f'while decoding table delta {code}::{contract_delta["table"]}') - raise e - + [maybe_decode_contract_row(row) for row in rows] break if result['traces'] and not len(action_whitelist) == 0: tx_traces = abi_unpack( 'transaction_trace[]', result['traces']) - for tx_trace in tx_traces: - for act_trace in tx_trace['action_traces']: - action = act_trace['act'] - account = action['account'] - name = action['name'] - - if account == 'eosio' and name == 'onblock': - continue - - try: - relevant_actions = action_whitelist.get(account, []) - if name not in relevant_actions: - continue - - contract_abi = contracts.get(account, None) - if not contract_abi: - raise SerializationException(f'abi for contract {account} not found') - - act_data = abi_unpack(name, action['data'], abi=contract_abi) - action['data'] = act_data - decoded_actions.append(action) - - except Exception as e: - e.add_note(f'while decoding action trace {account}::{name}') - raise e + [maybe_decode_tx_trace(tx_trace) for tx_trace in tx_traces] except Exception as e: e.add_note(f'while decoding block {block_num}') @@ -171,7 +177,7 @@ async def open_state_history( await ws.send_message(get_blocks_msg) # receive blocks & manage acks - acked_block = max_messages_in_flight + 1 + acked_block = start_block_num block_num = start_block_num - 1 while block_num != end_block_num: # receive get_blocks_result @@ -183,8 +189,8 @@ async def open_state_history( ) block_num = block.block_num - deltas_num = len(block.deltas) - actions_num = len(block.actions) + # deltas_num = len(block.deltas) + # actions_num = len(block.actions) # print(f'[{block_num}]: deltas: {deltas_num}, actions: {actions_num}') yield block