Add cli command group

README.md

@@ -3,52 +3,58 @@
 [![Join the chat at https://gitter.im/ethereum-eth](https://badges.gitter.im/ethereum-etl.svg)](https://gitter.im/ethereum-etl/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![Build Status](https://travis-ci.org/blockchain-etl/ethereum-etl.png)](https://travis-ci.org/blockchain-etl/ethereum-etl)
 
+Install Ethereum ETL:
+
+```bash
+pip install ethereumetl
+```
+
 Export blocks and transactions ([Schema](#blockscsv), [Reference](#export_blocks_and_transactionspy)):
 
 ```bash
-> python export_blocks_and_transactions.py --start-block 0 --end-block 500000 \
+> ethereumetl export_blocks_and_transactions --start-block 0 --end-block 500000 \
 --provider-uri https://mainnet.infura.io --blocks-output blocks.csv --transactions-output transactions.csv
 ```
 
 Export ERC20 and ERC721 transfers ([Schema](#token_transferscsv), [Reference](#export_token_transferspy)):
 
 ```bash
-> python export_token_transfers.py --start-block 0 --end-block 500000 \
+> ethereumetl export_token_transfers --start-block 0 --end-block 500000 \
 --provider-uri file://$HOME/Library/Ethereum/geth.ipc --output token_transfers.csv
 ```
 
 Export receipts and logs ([Schema](#receiptscsv), [Reference](#export_receipts_and_logspy)):
 
 ```bash
-> python export_receipts_and_logs.py --transaction-hashes transaction_hashes.txt \
+> ethereumetl export_receipts_and_logs --transaction-hashes transaction_hashes.txt \
 --provider-uri https://mainnet.infura.io --receipts-output receipts.csv --logs-output logs.csv
 ```
 
 Export ERC20 and ERC721 token details ([Schema](#tokenscsv), [Reference](#export_tokenspy)):
 
 ```bash
-> python export_tokens.py --token-addresses token_addresses.csv \
+> ethereumetl export_tokens --token-addresses token_addresses.csv \
 --provider-uri https://mainnet.infura.io --output tokens.csv
 ```
 
 Export traces ([Schema](#tracescsv), [Reference](#export_tracespy)):
 
 ```bash
-> python export_traces.py --start-block 0 --end-block 500000 \
+> ethereumetl export_traces --start-block 0 --end-block 500000 \
 --provider-uri file://$HOME/Library/Ethereum/parity.ipc --output traces.csv
 ```
 
 Export geth traces ([Reference](#export_geth_tracespy)):
 
 ```bash
-> python export_geth_traces.py --start-block 1 --end-block 500000 \
+> ethereumetl export_geth_traces --start-block 1 --end-block 500000 \
 --provider-uri file://$HOME/Library/Ethereum/geth.ipc --output geth_traces.json
 ```
 
 Extract geth traces ([Schema](#tracescsv), [Reference](#extract_geth_tracespy)):
 
 ```bash
-> python extract_geth_traces.py --input geth_traces.json --output traces.csv
+> ethereumetl extract_geth_traces --input geth_traces.json --output traces.csv
 ```
 
 [LIMITATIONS](#limitations)

ethereumetl.py

@@ -0,0 +1,26 @@
+# MIT License
+#
+# Copyright (c) 2018 Evgeny Medvedev, [email protected]
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+from ethereumetl.cli import cli
+
+cli()

ethereumetl/cli/__init__.py

@@ -0,0 +1,51 @@
+# MIT License
+#
+# Copyright (c) 2018 Evgeny Medvedev, [email protected]
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import click
+
+from ethereumetl.cli import export_blocks_and_transactions, export_receipts_and_logs, export_all, \
+    export_token_transfers, extract_token_transfers, export_contracts, export_tokens, get_block_range_for_date, \
+    get_block_range_for_timestamps, get_keccak_hash, export_traces
+
+
+@click.group()
+@click.pass_context
+def cli(ctx):
+    pass
+
+
+# export
+cli.add_command(export_all.cli, "export_all")
+cli.add_command(export_blocks_and_transactions.cli, "export_blocks_and_transactions")
+cli.add_command(export_receipts_and_logs.cli, "export_receipts_and_logs")
+cli.add_command(export_token_transfers.cli, "export_token_transfers")
+cli.add_command(extract_token_transfers.cli, "extract_token_transfers")
+cli.add_command(export_contracts.cli, "export_contracts")
+cli.add_command(export_tokens.cli, "export_tokens")
+cli.add_command(export_traces.cli, "export_traces")
+# cli.add_command(export_geth_traces.cli, "export_geth_traces")
+# cli.add_command(extract_geth_traces.cli, "extract_geth_traces")
+
+# utils
+cli.add_command(get_block_range_for_date.cli, "get_block_range_for_date")
+cli.add_command(get_block_range_for_timestamps.cli, "get_block_range_for_timestamps")
+cli.add_command(get_keccak_hash.cli, "get_keccak_hash")
+

ethereumetl/cli/export_all.py

@@ -25,9 +25,9 @@
 import re
 
 from datetime import datetime, timedelta
-from export_all_common import export_all_common
 from web3 import Web3
 
+from ethereumetl.jobs.export_all_common import export_all_common
 from ethereumetl.providers.auto import get_provider_from_uri
 from ethereumetl.service.eth_service import EthService
 
@@ -106,7 +106,7 @@ def get_partitions(start, end, partition_batch_size, provider_uri):
                    'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io')
 @click.option('-o', '--output-dir', default='output', type=str, help='Output directory, partitioned in Hive style.')
 @click.option('-w', '--max-workers', default=5, type=int, help='The maximum number of workers.')
-@click.option('-B', '--export-batch-size', default=100, type=int, help='The number of rows to write concurrently.')
+@click.option('-B', '--export-batch-size', default=100, type=int, help='The number of requests in JSON RPC batches.')
 def cli(start, end, partition_batch_size, provider_uri, output_dir, max_workers, export_batch_size):
     """Export all for a range of blocks."""
     export_all_common(get_partitions(start, end, partition_batch_size, provider_uri),

setup.py

@@ -1,7 +1,13 @@
+import os
+
 from setuptools import setup, find_packages
 
-with open('README.md', 'r') as fh:
-    long_description = fh.read()
+
+def read(fname):
+    return open(os.path.join(os.path.dirname(__file__), fname)).read()
+
+
+long_description = read('README.md') if os.path.isfile("README.md") else ""
 
 with open('requirements.txt') as f:
     requirements = f.read().splitlines()
@@ -21,25 +27,14 @@
         'Intended Audience :: Developers',
         'License :: OSI Approved :: MIT License',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
     ],
     keywords='ethereum',
-    python_requires='>=3.5.0,<3.7.0',
+    python_requires='>=3.6.0,<3.7.0',
     install_requires=requirements,
     entry_points={
         'console_scripts': [
-            'export_all=ethereumetl.cli.export_all:cli',
-            'export_blocks_and_transactions=ethereumetl.cli.export_blocks_and_transactions:cli',
-            'export_contracts=ethereumetl.cli.export_contracts:cli',
-            'export_receipts_and_logs=ethereumetl.cli.export_receipts_and_logs:cli',
-            'export_token_transfers=ethereumetl.cli.export_token_transfers:cli',
-            'export_tokens=ethereumetl.cli.export_tokens:cli',
-            'export_traces=ethereumetl.cli.export_traces:cli',
-            'extract_token_transfers=ethereumetl.cli.extract_token_transfers:cli',
-            'get_block_range_for_date=ethereumetl.cli.get_block_range_for_date:cli',
-            'get_block_range_for_timestamps=ethereumetl.cli.get_block_range_for_timestamps:cli',
-            'get_keccak_hash=ethereumetl.cli.get_keccak_hash:cli',
+            'ethereumetl=ethereumetl.cli:cli',
         ],
     },
     project_urls={