Merge pull request #58 from Synthetixio/extractor-configs

Extractor Config
Synthetixio · Jun 10, 2024 · de34726 · de34726
2 parents 18fd278 + 7156be9
commit de34726
Show file tree

Hide file tree

Showing 8 changed files with 147 additions and 141 deletions.
diff --git a/Makefile b/Makefile
@@ -18,7 +18,10 @@ build:
 	docker compose build transformer
 
 extract:
-	docker compose run extractors python main.py
+	docker compose run extractors python main.py configs/base_mainnet.yaml
+	docker compose run extractors python main.py configs/base_sepolia.yaml
+	docker compose run extractors python main.py configs/arbitrum_mainnet.yaml
+	docker compose run extractors python main.py configs/arbitrum_sepolia.yaml
 
 wrap:
 	docker compose run transformer python scripts/wrap_tables.py

diff --git a/extractors/.env.example b/extractors/.env.example
@@ -5,6 +5,3 @@ BASE_MAINNET_RPC=
 BASE_SEPOLIA_RPC=
 ARBITRUM_SEPOLIA_RPC=
 ARBITRUM_MAINNET_RPC=
-
-# settings
-REQUESTS_PER_SECOND=25
diff --git a/extractors/configs/arbitrum_mainnet.yaml b/extractors/configs/arbitrum_mainnet.yaml
@@ -0,0 +1,27 @@
+network_id: 42161
+
+blocks:
+  min_block: "218M"
+  requests_per_second: 25
+  block_increment: 4000
+
+eth_calls:
+  - contract_name: "CoreProxy"
+    function_name: "getVaultCollateral"
+    inputs:
+      - [1, "0x82aF49447D8a07e3bd95BD0d56f35241523fBab1"]
+      - [1, "0x912CE59144191C1204E64559FE8253a0e49E6548"]
+      - [1, "0xaf88d065e77c8cC2239327C5EDb3A432268e5831"]
+    min_block: "218M"
+    requests_per_second: 25
+    block_increment: 4000
+
+  - contract_name: "CoreProxy"
+    function_name: "getVaultDebt"
+    inputs:
+      - [1, "0x82aF49447D8a07e3bd95BD0d56f35241523fBab1"]
+      - [1, "0x912CE59144191C1204E64559FE8253a0e49E6548"]
+      - [1, "0xaf88d065e77c8cC2239327C5EDb3A432268e5831"]
+    min_block: "218M"
+    requests_per_second: 25
+    block_increment: 4000
diff --git a/extractors/configs/arbitrum_sepolia.yaml b/extractors/configs/arbitrum_sepolia.yaml
@@ -0,0 +1,30 @@
+network_id: 421614
+
+blocks:
+  min_block: "41M"
+  requests_per_second: 25
+  block_increment: 4000
+
+eth_calls:
+  - contract_name: "CoreProxy"
+    function_name: "getVaultCollateral"
+    inputs:
+      - [1, "0x980B62Da83eFf3D4576C647993b0c1D7faf17c73"]
+      - [1, "0x7b356eEdABc1035834cd1f714658627fcb4820E3"]
+      - [1, "0x75faf114eafb1BDbe2F0316DF893fd58CE46AA4d"]
+      - [1, "0xda7b438d762110083602AbC497b1Ec8Bc6605eC9"]
+    min_block: "41M"
+    requests_per_second: 25
+    block_increment: 4000
+
+
+  - contract_name: "CoreProxy"
+    function_name: "getVaultDebt"
+    inputs:
+      - [1, "0x980B62Da83eFf3D4576C647993b0c1D7faf17c73"]
+      - [1, "0x7b356eEdABc1035834cd1f714658627fcb4820E3"]
+      - [1, "0x75faf114eafb1BDbe2F0316DF893fd58CE46AA4d"]
+      - [1, "0xda7b438d762110083602AbC497b1Ec8Bc6605eC9"]
+    min_block: "41M"
+    requests_per_second: 25
+    block_increment: 4000
diff --git a/extractors/configs/base_mainnet.yaml b/extractors/configs/base_mainnet.yaml
@@ -0,0 +1,20 @@
+network_id: 8453
+
+blocks:
+  min_block: "7.5M"
+  requests_per_second: 25
+
+eth_calls:
+  - contract_name: "CoreProxy"
+    function_name: "getVaultCollateral"
+    inputs:
+      - [1, "0xC74eA762cF06c9151cE074E6a569a5945b6302E7"]
+    min_block: "7.5M"
+    requests_per_second: 25
+
+  - contract_name: "CoreProxy"
+    function_name: "getVaultDebt"
+    inputs:
+      - [1, "0xC74eA762cF06c9151cE074E6a569a5945b6302E7"]
+    min_block: "7.5M"
+    requests_per_second: 25
diff --git a/extractors/configs/base_sepolia.yaml b/extractors/configs/base_sepolia.yaml
@@ -0,0 +1,20 @@
+network_id: 84532
+
+blocks:
+  min_block: "8M"
+  requests_per_second: 25
+
+eth_calls:
+  - contract_name: "CoreProxy"
+    function_name: "getVaultCollateral"
+    inputs:
+      - [1, "0x8069c44244e72443722cfb22DcE5492cba239d39"]
+    min_block: "8M"
+    requests_per_second: 25
+
+  - contract_name: "CoreProxy"
+    function_name: "getVaultDebt"
+    inputs:
+      - [1, "0x8069c44244e72443722cfb22DcE5492cba239d39"]
+    min_block: "8M"
+    requests_per_second: 25
diff --git a/extractors/main.py b/extractors/main.py
@@ -1,148 +1,56 @@
 import os
+import yaml
+import argparse
 from dotenv import load_dotenv
-from src.extract import extract_data, extract_blocks
+from src.extract import extract_blocks, extract_data
 
 # load environment variables
 load_dotenv()
 
-REQUESTS_PER_SECOND = int(os.getenv("REQUESTS_PER_SECOND"))
+# parse command-line arguments
+parser = argparse.ArgumentParser(description="Extract data from Ethereum nodes.")
+parser.add_argument("config", help="Path to the YAML configuration file")
+parser.add_argument("--name", help="Name of the configuration to use (optional)")
+args = parser.parse_args()
 
-# extract blocks
-block_inputs = [
-    # base mainnet
-    {
-        "network_id": 8453,
-        "min_block": "7.5M",
-        "requests_per_second": REQUESTS_PER_SECOND,
-    },
-    # base sepolia
-    {
-        "network_id": 84532,
-        "min_block": "8M",
-        "requests_per_second": 25,
-    },
-    # arbitrum sepolia
-    {
-        "network_id": 421614,
-        "min_block": "41M",
-        "block_increment": 4000,
-    },
-    # arbitrum mainnet
-    {
-        "network_id": 42161,
-        "min_block": "218M",
-        "block_increment": 4000,
-    },
-]
+# load configurations from YAML file
+with open(args.config, "r") as f:
+    config = yaml.safe_load(f)
 
-for block_input in block_inputs:
-    extract_blocks(**block_input)
+network_id = config.get("network_id")
+block_config = config.get("blocks")
+eth_call_configs = config.get("eth_calls", [])
 
-
-# extract eth_call data
-func_inputs = [
-    # base mainnet
-    {
-        "network_id": 8453,
-        "contract_name": "CoreProxy",
-        "function_name": "getVaultCollateral",
-        "inputs": [
-            (1, "0xC74eA762cF06c9151cE074E6a569a5945b6302E7"),
-        ],
-        "min_block": "7.5M",
-        "requests_per_second": REQUESTS_PER_SECOND,
-    },
-    {
-        "network_id": 8453,
-        "contract_name": "CoreProxy",
-        "function_name": "getVaultDebt",
-        "inputs": [
-            (1, "0xC74eA762cF06c9151cE074E6a569a5945b6302E7"),
-        ],
-        "min_block": "7.5M",
-        "requests_per_second": REQUESTS_PER_SECOND,
-    },
-    # base sepolia
-    {
-        "network_id": 84532,
-        "contract_name": "CoreProxy",
-        "function_name": "getVaultCollateral",
-        "inputs": [
-            (1, "0x8069c44244e72443722cfb22DcE5492cba239d39"),
-        ],
-        "min_block": "8M",
-        "requests_per_second": 25,
-    },
-    {
-        "network_id": 84532,
-        "contract_name": "CoreProxy",
-        "function_name": "getVaultDebt",
-        "inputs": [
-            (1, "0x8069c44244e72443722cfb22DcE5492cba239d39"),
-        ],
-        "min_block": "8M",
-        "requests_per_second": 25,
-    },
-    # arbitrum sepolia
-    {
-        "network_id": 421614,
-        "contract_name": "CoreProxy",
-        "function_name": "getVaultCollateral",
-        "inputs": [
-            (1, "0x980B62Da83eFf3D4576C647993b0c1D7faf17c73"),  # WETH
-            (1, "0x7b356eEdABc1035834cd1f714658627fcb4820E3"),  # ARB
-            (1, "0x75faf114eafb1BDbe2F0316DF893fd58CE46AA4d"),  # USDC
-            (1, "0xda7b438d762110083602AbC497b1Ec8Bc6605eC9"),  # DAI
-        ],
-        "min_block": "41M",
-        "requests_per_second": 25,
-        "block_increment": 4000,
-    },
-    {
-        "network_id": 421614,
-        "contract_name": "CoreProxy",
-        "function_name": "getVaultDebt",
-        "inputs": [
-            (1, "0x980B62Da83eFf3D4576C647993b0c1D7faf17c73"),  # WETH
-            (1, "0x7b356eEdABc1035834cd1f714658627fcb4820E3"),  # ARB
-            (1, "0x75faf114eafb1BDbe2F0316DF893fd58CE46AA4d"),  # USDC
-            (1, "0xda7b438d762110083602AbC497b1Ec8Bc6605eC9"),  # DAI
-        ],
-        "min_block": "41M",
-        "requests_per_second": 25,
-        "block_increment": 4000,
-    },
-    # arbitrum mainnet
-    {
-        "network_id": 42161,
-        "contract_name": "CoreProxy",
-        "function_name": "getVaultCollateral",
-        "inputs": [
-            (1, "0x82aF49447D8a07e3bd95BD0d56f35241523fBab1"),  # WETH
-            (1, "0x912CE59144191C1204E64559FE8253a0e49E6548"),  # ARB
-            (1, "0xaf88d065e77c8cC2239327C5EDb3A432268e5831"),  # USDC
-        ],
-        "min_block": "218M",
-        "requests_per_second": 25,
-        "block_increment": 4000,
-    },
-    {
-        "network_id": 42161,
-        "contract_name": "CoreProxy",
-        "function_name": "getVaultDebt",
-        "inputs": [
-            (1, "0x82aF49447D8a07e3bd95BD0d56f35241523fBab1"),  # WETH
-            (1, "0x912CE59144191C1204E64559FE8253a0e49E6548"),  # ARB
-            (1, "0xaf88d065e77c8cC2239327C5EDb3A432268e5831"),  # USDC
-        ],
-        "min_block": "218M",
-        "requests_per_second": 25,
-        "block_increment": 4000,
-    },
-]
-
-for func_input in func_inputs:
+# determine the flow based on the --name argument
+if args.name:
+    if args.name == "blocks":
+        # run blocks only
+        try:
+            extract_blocks(network_id=network_id, **block_config)
+        except Exception as e:
+            print(f"Error extracting blocks: {e}")
+    else:
+        # run the specified eth_call only
+        eth_call_config = next(
+            (ec for ec in eth_call_configs if ec["function_name"] == args.name), None
+        )
+        if eth_call_config:
+            try:
+                extract_data(network_id=network_id, **eth_call_config)
+            except Exception as e:
+                print(f"Error extracting eth_call {args.name}: {e}")
+        else:
+            print(f"No configuration found with name {args.name}")
+else:
+    # run everything
     try:
-        extract_data(**func_input)
+        extract_blocks(network_id=network_id, **block_config)
     except Exception as e:
-        print(f"Error: {e} for inputs {func_input}")
+        print(f"Error extracting blocks: {e}")
+
+    for eth_call_config in eth_call_configs:
+        try:
+            extract_data(network_id=network_id, **eth_call_config)
+        except Exception as e:
+            print(f"Error extracting eth_call {eth_call_config.get('name')}: {e}")
+            continue
diff --git a/extractors/requirements.txt b/extractors/requirements.txt
@@ -1,4 +1,5 @@
 python-dotenv
+PyYAML
 pyarrow
 duckdb==0.10.2
 polars-lts-cpu