Add blocks and transactions test cases

.travis.yml

@@ -4,4 +4,4 @@ python:
 install:
   - travis_retry pip install -r requirements.txt
 script:
-  - pytest
+  - pytest -vv

ethereumetl/ipc.py

@@ -13,6 +13,20 @@
     JSONDecodeError = ValueError
 
 
+class ThreadLocalIPCWrapper:
+    def __init__(self, ipc_wrapper_factory):
+        self.ipc_wrapper_factory = ipc_wrapper_factory
+        self.thread_local = threading.local()
+
+    def make_request(self, text):
+        return self._get_ipc_wrapper().make_request(text)
+
+    def _get_ipc_wrapper(self):
+        if getattr(self.thread_local, 'ipc_wrapper', None) is None:
+            self.thread_local.ipc_wrapper = self.ipc_wrapper_factory()
+        return self.thread_local.ipc_wrapper
+
+
 # Mostly copied from web3.py/providers/ipc.py. Supports batch requests.
 # Will be removed once batch feature is added to web3.py https://github.com/ethereum/web3.py/issues/832
 class IPCWrapper:

ethereumetl/job/export_blocks_job.py

@@ -1,5 +1,4 @@
 import json
-import threading
 
 from web3.utils.threads import Timeout
 
@@ -20,15 +19,15 @@ def __init__(
             start_block,
             end_block,
             batch_size,
-            ipc_wrapper_factory,
+            ipc_wrapper,
             max_workers=5,
             max_queue=5,
             blocks_output=None,
             transactions_output=None):
         self.start_block = start_block
         self.end_block = end_block
         self.batch_size = batch_size
-        self.ipc_wrapper_factory = ipc_wrapper_factory
+        self.ipc_wrapper = ipc_wrapper
         self.max_workers = max_workers
         self.max_queue = max_queue
         self.blocks_output = blocks_output
@@ -49,7 +48,6 @@ def __init__(
         self.transactions_exporter = None
 
         self.executor = None
-        self.thread_local = threading.local()
         self.futures = []
 
     def _start(self):
@@ -79,17 +77,12 @@ def _export_batch_with_retries(self, batch_start, batch_end):
 
     def _export_batch(self, batch_start, batch_end):
         blocks_rpc = list(generate_get_block_by_number_json_rpc(batch_start, batch_end, self.export_transactions))
-        response = self._get_ipc_wrapper().make_request(json.dumps(blocks_rpc))
+        response = self.ipc_wrapper.make_request(json.dumps(blocks_rpc))
         for response_item in response:
             result = response_item['result']
             block = self.block_mapper.json_dict_to_block(result)
             self._export_block(block)
 
-    def _get_ipc_wrapper(self):
-        if getattr(self.thread_local, 'ipc_wrapper', None) is None:
-            self.thread_local.ipc_wrapper = self.ipc_wrapper_factory()
-        return self.thread_local.ipc_wrapper
-
     def _check_completed_futures(self):
         for future in self.futures.copy():
             if future.done():

export_blocks_and_transactions.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 import argparse
 
-from ethereumetl.ipc import IPCWrapper
+from ethereumetl.ipc import IPCWrapper, ThreadLocalIPCWrapper
 from ethereumetl.job.export_blocks_job import ExportBlocksJob
 
 parser = argparse.ArgumentParser(description='Export blocks and transactions.')
@@ -25,7 +25,7 @@
     start_block=args.start_block,
     end_block=args.end_block,
     batch_size=args.batch_size,
-    ipc_wrapper_factory=lambda: IPCWrapper(args.ipc_path, args.ipc_timeout),
+    ipc_wrapper=ThreadLocalIPCWrapper(lambda: IPCWrapper(args.ipc_path, args.ipc_timeout)),
     max_workers=args.max_workers,
     max_queue=args.max_queue,
     blocks_output=args.blocks_output,

tests/ethereumetl/job/__init__.py

@@ -0,0 +1,4 @@
+import pytest
+
+# we want to have pytest assert introspection in the helpers
+pytest.register_assert_rewrite('tests.helpers')

tests/ethereumetl/job/fixtures/block_without_transactions/expected_blocks.csv

@@ -0,0 +1,2 @@
+block_number,block_hash,block_parent_hash,block_nonce,block_sha3_uncles,block_logs_bloom,block_transactions_root,block_state_root,block_miner,block_difficulty,block_total_difficulty,block_size,block_extra_data,block_gas_limit,block_gas_used,block_timestamp,block_transaction_count
+0,0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3,0x0000000000000000000000000000000000000000000000000000000000000000,0x0000000000000042,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421,0xd7f8974fb5ac78d9ac099b9ad5018bedc2ce0a72dad1827a1709da30580f0544,0x0000000000000000000000000000000000000000,17179869184,17179869184,540,0x11bbe8db4e347b4e8c937c1c8370e4b5ed33adb3db69cbdb7a38e1e50b1b82fa,5000,0,0,0

tests/ethereumetl/job/fixtures/block_without_transactions/ipc_response.0.json

@@ -0,0 +1,25 @@
+{
+  "result": {
+    "author": "0x0000000000000000000000000000000000000000",
+    "difficulty": "0x400000000",
+    "extraData": "0x11bbe8db4e347b4e8c937c1c8370e4b5ed33adb3db69cbdb7a38e1e50b1b82fa",
+    "gasLimit": "0x1388",
+    "gasUsed": "0x0",
+    "hash": "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3",
+    "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+    "miner": "0x0000000000000000000000000000000000000000",
+    "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+    "nonce": "0x0000000000000042",
+    "number": "0x0",
+    "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+    "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+    "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
+    "size": "0x21c",
+    "stateRoot": "0xd7f8974fb5ac78d9ac099b9ad5018bedc2ce0a72dad1827a1709da30580f0544",
+    "timestamp": "0x0",
+    "totalDifficulty": "0x400000000",
+    "transactions": [],
+    "transactionsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+    "uncles": []
+  }
+}

tests/ethereumetl/job/fixtures/blocks_with_transactions/expected_blocks.csv

@@ -0,0 +1,3 @@
+block_number,block_hash,block_parent_hash,block_nonce,block_sha3_uncles,block_logs_bloom,block_transactions_root,block_state_root,block_miner,block_difficulty,block_total_difficulty,block_size,block_extra_data,block_gas_limit,block_gas_used,block_timestamp,block_transaction_count
+47218,0x889c421abc62a48641eee140519e6da8c9dc01d85d8f5c4fbc3c13e3c6e4cb3e,0xfc1dd3249585b593ad18822a95873c75779ccbe8a420b457be380b977fc38e87,0xfaa1e51d379b21f7,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,0xdef39a3afcfb55fabd16d7d5a19d8b078cb95a117ea9596af289525eb8592982,0xab6d1b109c04c1f43f86da33ece31689d9ba1c44ea809d0979c05057a0703c28,0x9746c7e1ef2bd21ff3997fa467593a89cb852bd0,1460233976906,44246932724217368,766,0x476574682f76312e302e312f77696e646f77732f676f312e342e32,42085,42000,1438936285,2
+47219,0x944f09177142833c644c979a83900d8cae1ee67369774b88b3b330bb72825082,0x889c421abc62a48641eee140519e6da8c9dc01d85d8f5c4fbc3c13e3c6e4cb3e,0x52cf720359834975,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,0xd757552351d6a714feda148f8f6f1283e000e05ea407352d45bc9dfc4c16ffe9,0x05a16e52dbacec805dc881439ef54338e8324ee133a4dbbb8ab17f8c73290054,0xf927a40c8b7f6e07c5af7fa2155b4864a4112b13,1459520972035,44248392245189403,763,0x476574682f76312e302e312f6c696e75782f676f312e342e32,42125,42000,1438936326,2

tests/ethereumetl/job/fixtures/blocks_with_transactions/expected_transactions.csv

@@ -0,0 +1,5 @@
+tx_hash,tx_nonce,tx_block_hash,tx_block_number,tx_index,tx_from,tx_to,tx_value,tx_gas,tx_gas_price,tx_input
+0xbd5ab8937e52a6244209d804471be4878df6c364bca0111dd6d05e0d3edf63cf,79,0x944f09177142833c644c979a83900d8cae1ee67369774b88b3b330bb72825082,47219,0,0xe6a7a1d47ff21b6321162aea7c6cb457d5476bca,0xe25e3a1947405a1f82dd8e3048a9ca471dc782e1,8306052477120672000,21000,61580653163,0x
+0x4bcc1dd0c56c0b767b1ee3cb8bce7df44518f1696205299e34eb53a5e00a863e,1,0x944f09177142833c644c979a83900d8cae1ee67369774b88b3b330bb72825082,47219,1,0xf9a19aea1193d9b9e4ef2f5b8c9ec8df93a22356,0x32be343b94f860124dc4fee278fdcbd38c102d88,1998716170000000000,21000,61134768794,0x
+0x99f1097abd8f33a68f0ed63d60de5f3e7e2a3e0579b90d5f46a4f201c658b46d,9,0x889c421abc62a48641eee140519e6da8c9dc01d85d8f5c4fbc3c13e3c6e4cb3e,47218,0,0x1406854d149e081ac09cb4ca560da463f3123059,0xa0e74ae010d51894734c308d612131056bb721ad,110000000000000000000,40000,62227241854,0x
+0x95844e6c54b4aafc8e1f75784127529280e75c3a980d91f6dfca1c1b0eb078fb,78,0x889c421abc62a48641eee140519e6da8c9dc01d85d8f5c4fbc3c13e3c6e4cb3e,47218,1,0xe6a7a1d47ff21b6321162aea7c6cb457d5476bca,0xee80ef3c49d9465c7fc2b3d7373fdbbbc3fe282f,8140416390630760000,21000,62222792381,0x

tests/ethereumetl/job/fixtures/blocks_with_transactions/ipc_response.47218.json

@@ -0,0 +1,76 @@
+{
+  "jsonrpc": "2.0",
+  "result": {
+    "author": "0x9746c7e1ef2bd21ff3997fa467593a89cb852bd0",
+    "difficulty": "0x153fcba3c4a",
+    "extraData": "0x476574682f76312e302e312f77696e646f77732f676f312e342e32",
+    "gasLimit": "0xa465",
+    "gasUsed": "0xa410",
+    "hash": "0x889c421abc62a48641eee140519e6da8c9dc01d85d8f5c4fbc3c13e3c6e4cb3e",
+    "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+    "miner": "0x9746c7e1ef2bd21ff3997fa467593a89cb852bd0",
+    "mixHash": "0xfa4ab37bc8ef9ca8e95ca2c92135d5e5c4451b25040e454b6deb422c5f48e586",
+    "nonce": "0xfaa1e51d379b21f7",
+    "number": "0xb872",
+    "parentHash": "0xfc1dd3249585b593ad18822a95873c75779ccbe8a420b457be380b977fc38e87",
+    "receiptsRoot": "0xf0afaf08454f89092907f820c40db77afbd0270f179166907dc110f9723972d7",
+    "sealFields": [
+      "0xa0fa4ab37bc8ef9ca8e95ca2c92135d5e5c4451b25040e454b6deb422c5f48e586",
+      "0x88faa1e51d379b21f7"
+    ],
+    "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
+    "size": "0x2fe",
+    "stateRoot": "0xab6d1b109c04c1f43f86da33ece31689d9ba1c44ea809d0979c05057a0703c28",
+    "timestamp": "0x55c46cdd",
+    "totalDifficulty": "0x9d3259d36d7618",
+    "transactions": [
+      {
+        "blockHash": "0x889c421abc62a48641eee140519e6da8c9dc01d85d8f5c4fbc3c13e3c6e4cb3e",
+        "blockNumber": "0xb872",
+        "chainId": null,
+        "condition": null,
+        "creates": null,
+        "from": "0x1406854d149e081ac09cb4ca560da463f3123059",
+        "gas": "0x9c40",
+        "gasPrice": "0xe7d085b7e",
+        "hash": "0x99f1097abd8f33a68f0ed63d60de5f3e7e2a3e0579b90d5f46a4f201c658b46d",
+        "input": "0x",
+        "nonce": "0x9",
+        "publicKey": "0x20de362c2ce12c75da719cbc710f2d2efa35b7dc0da347fdc953dad40375457a93d9cdfa6d53131a4a29a9cf04bca7edc016dd7936cb7a9f9a465381c0d366ab",
+        "r": "0xd3c39a9c531e2c6eb4c11355d0e553ff8f39e136d121109dd7163b8ac759b5d9",
+        "raw": "0xf86d09850e7d085b7e829c4094a0e74ae010d51894734c308d612131056bb721ad8905f68e8131ecf80000801ba0d3c39a9c531e2c6eb4c11355d0e553ff8f39e136d121109dd7163b8ac759b5d9a06bd9643ce14a57db154a3e5fb1af12cd6854e00e3e82ceb8b1e06b336b44af93",
+        "s": "0x6bd9643ce14a57db154a3e5fb1af12cd6854e00e3e82ceb8b1e06b336b44af93",
+        "standardV": "0x0",
+        "to": "0xa0e74ae010d51894734c308d612131056bb721ad",
+        "transactionIndex": "0x0",
+        "v": "0x1b",
+        "value": "0x5f68e8131ecf80000"
+      },
+      {
+        "blockHash": "0x889c421abc62a48641eee140519e6da8c9dc01d85d8f5c4fbc3c13e3c6e4cb3e",
+        "blockNumber": "0xb872",
+        "chainId": null,
+        "condition": null,
+        "creates": null,
+        "from": "0xe6a7a1d47ff21b6321162aea7c6cb457d5476bca",
+        "gas": "0x5208",
+        "gasPrice": "0xe7cc476bd",
+        "hash": "0x95844e6c54b4aafc8e1f75784127529280e75c3a980d91f6dfca1c1b0eb078fb",
+        "input": "0x",
+        "nonce": "0x4e",
+        "publicKey": "0x9f0fc2dda3a37d2a7c5fa7ff5630551b06d88b7e9ff0c9ddbb59e3aa874f0438f5dd99d6c83fc2be5e46c8708fb099333a2ec046a31d31a75cf3c6c2e9be17bb",
+        "r": "0x575f2d6750ed42ef50f6fa356c27e3d4c42f7c97f5217f16fac794bbf05f84f6",
+        "raw": "0xf86c4e850e7cc476bd82520894ee80ef3c49d9465c7fc2b3d7373fdbbbc3fe282f8870f89193d7505e40801ca0575f2d6750ed42ef50f6fa356c27e3d4c42f7c97f5217f16fac794bbf05f84f6a02e709c44893f46894f687664d3d2cf396b7c8b2cb73e817cee25a68fdd1e76a0",
+        "s": "0x2e709c44893f46894f687664d3d2cf396b7c8b2cb73e817cee25a68fdd1e76a0",
+        "standardV": "0x1",
+        "to": "0xee80ef3c49d9465c7fc2b3d7373fdbbbc3fe282f",
+        "transactionIndex": "0x1",
+        "v": "0x1c",
+        "value": "0x70f89193d7505e40"
+      }
+    ],
+    "transactionsRoot": "0xdef39a3afcfb55fabd16d7d5a19d8b078cb95a117ea9596af289525eb8592982",
+    "uncles": []
+  },
+  "id": 1
+}

tests/ethereumetl/job/fixtures/blocks_with_transactions/ipc_response.47219.json

@@ -0,0 +1,76 @@
+{
+  "jsonrpc": "2.0",
+  "result": {
+    "author": "0xf927a40c8b7f6e07c5af7fa2155b4864a4112b13",
+    "difficulty": "0x153d23aa503",
+    "extraData": "0x476574682f76312e302e312f6c696e75782f676f312e342e32",
+    "gasLimit": "0xa48d",
+    "gasUsed": "0xa410",
+    "hash": "0x944f09177142833c644c979a83900d8cae1ee67369774b88b3b330bb72825082",
+    "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+    "miner": "0xf927a40c8b7f6e07c5af7fa2155b4864a4112b13",
+    "mixHash": "0x7db0f956effd39acc65cfd014daf743b3bdc1f4c2967cd30141570d61ae136d7",
+    "nonce": "0x52cf720359834975",
+    "number": "0xb873",
+    "parentHash": "0x889c421abc62a48641eee140519e6da8c9dc01d85d8f5c4fbc3c13e3c6e4cb3e",
+    "receiptsRoot": "0x9c8327de15d9d1668feb6e3583ddc337fb857620a5a3ff66d0d66148864b9d55",
+    "sealFields": [
+      "0xa07db0f956effd39acc65cfd014daf743b3bdc1f4c2967cd30141570d61ae136d7",
+      "0x8852cf720359834975"
+    ],
+    "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
+    "size": "0x2fb",
+    "stateRoot": "0x05a16e52dbacec805dc881439ef54338e8324ee133a4dbbb8ab17f8c73290054",
+    "timestamp": "0x55c46d06",
+    "totalDifficulty": "0x9d33ada5a81b1b",
+    "transactions": [
+      {
+        "blockHash": "0x944f09177142833c644c979a83900d8cae1ee67369774b88b3b330bb72825082",
+        "blockNumber": "0xb873",
+        "chainId": null,
+        "condition": null,
+        "creates": null,
+        "from": "0xe6a7a1d47ff21b6321162aea7c6cb457d5476bca",
+        "gas": "0x5208",
+        "gasPrice": "0xe567e326b",
+        "hash": "0xbd5ab8937e52a6244209d804471be4878df6c364bca0111dd6d05e0d3edf63cf",
+        "input": "0x",
+        "nonce": "0x4f",
+        "publicKey": "0x9f0fc2dda3a37d2a7c5fa7ff5630551b06d88b7e9ff0c9ddbb59e3aa874f0438f5dd99d6c83fc2be5e46c8708fb099333a2ec046a31d31a75cf3c6c2e9be17bb",
+        "r": "0x622261f6c8be0b208798e8ffbfdd5c7f34f1d9723887a5dc9414afe7d7e6da05",
+        "raw": "0xf86c4f850e567e326b82520894e25e3a1947405a1f82dd8e3048a9ca471dc782e188734506b87887f100801ca0622261f6c8be0b208798e8ffbfdd5c7f34f1d9723887a5dc9414afe7d7e6da05a0460b192f2383ac1066aabfea8ac0837362285bde92fecdb2e899e1194dc4c55f",
+        "s": "0x460b192f2383ac1066aabfea8ac0837362285bde92fecdb2e899e1194dc4c55f",
+        "standardV": "0x1",
+        "to": "0xe25e3a1947405a1f82dd8e3048a9ca471dc782e1",
+        "transactionIndex": "0x0",
+        "v": "0x1c",
+        "value": "0x734506b87887f100"
+      },
+      {
+        "blockHash": "0x944f09177142833c644c979a83900d8cae1ee67369774b88b3b330bb72825082",
+        "blockNumber": "0xb873",
+        "chainId": null,
+        "condition": null,
+        "creates": null,
+        "from": "0xf9a19aea1193d9b9e4ef2f5b8c9ec8df93a22356",
+        "gas": "0x5208",
+        "gasPrice": "0xe3bea8a9a",
+        "hash": "0x4bcc1dd0c56c0b767b1ee3cb8bce7df44518f1696205299e34eb53a5e00a863e",
+        "input": "0x",
+        "nonce": "0x1",
+        "publicKey": "0x04c65d5cc3631a511c5b899b5f7e059947774e2fc3a9cf2dd3e8783996bbc16fa0950a8355aee8f85e7c4e43928c3105be0213c210b153b5db2e9e01e1dd0527",
+        "r": "0x2f8ba5d5ababcbb358a38f1c2bc0b85213d380f32c92f68f78df90ad4e5a29f9",
+        "raw": "0xf86c01850e3bea8a9a8252089432be343b94f860124dc4fee278fdcbd38c102d88881bbcddc457ade400801ca02f8ba5d5ababcbb358a38f1c2bc0b85213d380f32c92f68f78df90ad4e5a29f9a07d6e356aee799027f6337378bde0443630b7776bfe73d215b2a74da2b5f572c2",
+        "s": "0x7d6e356aee799027f6337378bde0443630b7776bfe73d215b2a74da2b5f572c2",
+        "standardV": "0x1",
+        "to": "0x32be343b94f860124dc4fee278fdcbd38c102d88",
+        "transactionIndex": "0x1",
+        "v": "0x1c",
+        "value": "0x1bbcddc457ade400"
+      }
+    ],
+    "transactionsRoot": "0xd757552351d6a714feda148f8f6f1283e000e05ea407352d45bc9dfc4c16ffe9",
+    "uncles": []
+  },
+  "id": 1
+}