Merge pull request #603 from neonlabsorg/ci-test-block->master_0.6.0-rc4

ci_test_block -> master v0.6.0-rc4

.buildkite/pipeline.yml

@@ -18,6 +18,7 @@ steps:
       - "evm_loader.log"
       - "faucet.log"
       - "airdropper.log"
+      - "indexer.log"
 
   - wait
 

.buildkite/steps/build-image.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 REVISION=$(git rev-parse HEAD)
 
 set ${SOLANA_REVISION:=v1.8.12-testnet}
-set ${EVM_LOADER_REVISION:=stable}
+set ${EVM_LOADER_REVISION:=v0.6.0-rc3}
 
 # Refreshing neonlabsorg/solana:latest image is required to run .buildkite/steps/build-image.sh locally
 docker pull neonlabsorg/solana:${SOLANA_REVISION}

.buildkite/steps/deploy-test.sh

@@ -46,6 +46,7 @@ function cleanup_docker {
     if docker logs evm_loader >evm_loader.log 2>&1; then echo "evm_loader logs saved"; fi
     if docker logs faucet >faucet.log 2>&1; then echo "faucet logs saved"; fi
     if docker logs airdropper >airdropper.log 2>&1; then echo "airdropper logs saved"; fi
+    if docker logs indexer >indexer.log 2>&1; then echo "indexer logs saved"; fi
 
     echo "\nCleanup docker-compose..."
     docker-compose -f proxy/docker-compose-test.yml down
@@ -73,17 +74,19 @@ echo "Wait proxy..." && wait-for-proxy "$PROXY_URL"
 
 export EVM_LOADER=$(docker exec proxy bash -c "solana address -k /spl/bin/evm_loader-keypair.json")
 export SOLANA_URL=$(docker exec solana bash -c 'echo "$SOLANA_URL"')
+export FAUCET_URL=$(docker exec proxy bash -c 'echo "$FAUCET_URL"')
 
 echo "EVM_LOADER" $EVM_LOADER
 echo "SOLANA_URL" $SOLANA_URL
+echo "FAUCET_URL" $FAUCET_URL
 
 echo "Run proxy tests..."
 docker run --rm -ti --network=container:proxy \
      -e PROXY_URL \
      -e EVM_LOADER \
      -e SOLANA_URL \
+     -e FAUCET_URL \
      -e EXTRA_GAS=100000 \
-     -e NEW_USER_AIRDROP_AMOUNT=100 \
      -e POSTGRES_DB=neon-db \
      -e POSTGRES_USER=neon-proxy \
      -e POSTGRES_PASSWORD=neon-proxy-pass \
@@ -93,7 +96,8 @@ docker run --rm -ti --network=container:proxy \
      $PROXY_IMAGE
 
 echo "Run uniswap-v2-core tests..."
-docker run --rm -ti --network=host \
+docker run --rm -ti --network=container:proxy \
+     -e FAUCET_URL \
      --entrypoint ./deploy-test.sh \
      ${EXTRA_ARGS:-} \
      $UNISWAP_V2_CORE_IMAGE \

Dockerfile

@@ -18,8 +18,9 @@ RUN apt update && \
     pip3 install --upgrade pip && \
     /bin/bash -c "source venv/bin/activate" && \
     pip install -r requirements.txt && \
+    pip3 install py-solc-x && \
+    python3 -c "import solcx; solcx.install_solc(version='0.7.6')" && \
     apt remove -y git && \
-    pip install py-solc-x && \
     rm -rf /var/lib/apt/lists/*
 
 COPY --from=cli /opt/solana/bin/solana \

log_cfg.json

@@ -24,12 +24,12 @@
     },
     "formatters": {
         "standard": {
-            "format": "%(asctime)s.%(msecs)03d %(levelname)8s %(process)6d:%(threadName)-11s %(name)15s:%(class)-30s %(context)s %(message)s",
+            "format": "%(asctime)s.%(msecs)03d %(levelname)-.1s %(filename)s:%(lineno)d %(process)d %(name)s:%(class)s %(context)s %(message)s",
             "style": "%",
             "datefmt": "%Y-%m-%d %H:%M:%S"
         },
         "root": {
-            "format": "%(asctime)s.%(msecs)03d %(levelname)8s %(process)6d:%(threadName)-11s %(name)15s:Undefined                      %(context)s %(message)s",
+            "format": "%(asctime)s.%(msecs)03d %(levelname)-.1s %(filename)s:%(lineno)d %(process)d %(name)s:Undefined %(context)s %(message)s",
             "style": "%",
             "datefmt": "%Y-%m-%d %H:%M:%S"
         }

pretty_log_cfg.json

@@ -24,12 +24,12 @@
     },
     "formatters": {
         "standard": {
-            "format": "%(asctime)s.%(msecs)03d [%(levelname)-.1s] pid:%(process)d %(name)s:%(filename)s:%(lineno)d %(context)s %(message)s",
+            "format": "%(asctime)s.%(msecs)03d [%(levelname)-.1s] P:%(process)d %(name)s:%(class)s:%(funcName)s:%(lineno)d %(context)s %(message)s",
             "style": "%",
             "datefmt": "%Y-%m-%d %H:%M:%S"
         },
         "root": {
-            "format": "%(asctime)s.%(msecs)03d [%(levelname)-.1s] pid:%(process)d %(name)s:Undefined %(context)s %(message)s",
+            "format": "%(asctime)s.%(msecs)03d [%(levelname)-.1s] P:%(process)d %(name)s:%(lineno)d %(context)s %(message)s",
             "style": "%",
             "datefmt": "%Y-%m-%d %H:%M:%S"
         }

proxy/__init__.py

@@ -11,7 +11,6 @@
 from .proxy import entry_point
 from .proxy import main, start
 from .proxy import Proxy
-from .testing.test_case import TestCase
 
 __all__ = [
     # PyPi package entry_point. See
@@ -20,8 +19,5 @@
     # Embed proxy.py. See
     # https://github.com/abhinavsingh/proxy.py#embed-proxypy
     'main', 'start',
-    # Unit testing with proxy.py. See
-    # https://github.com/abhinavsingh/proxy.py#unit-testing-with-proxypy
-    'TestCase',
     'Proxy',
 ]

proxy/__main__.py

@@ -13,14 +13,14 @@
 from .proxy import entry_point
 import os
 from .indexer.airdropper import run_airdropper
-from solana.rpc.api import Client
+from .indexer.indexer import run_indexer
 
 if __name__ == '__main__':
     airdropper_mode = os.environ.get('AIRDROPPER_MODE', 'False').lower() in [1, 'true', 'True']
+    indexer_mode = os.environ.get('INDEXER_MODE', 'False').lower() in [1, 'true', 'True']
     if airdropper_mode:
         print("Will run in airdropper mode")
         solana_url = os.environ['SOLANA_URL']
-        evm_loader_id = os.environ['EVM_LOADER']
         pyth_mapping_account = PublicKey(os.environ['PYTH_MAPPING_ACCOUNT'])
         faucet_url = os.environ['FAUCET_URL']
         wrapper_whitelist = os.environ['INDEXER_ERC20_WRAPPER_WHITELIST']
@@ -32,12 +32,17 @@
         max_conf = float(os.environ.get('MAX_CONFIDENCE_INTERVAL', 0.02))
 
         run_airdropper(solana_url,
-                       evm_loader_id,
                        pyth_mapping_account,
                        faucet_url,
                        wrapper_whitelist,
                        neon_decimals,
                        pp_solana_url,
                        max_conf)
+    elif indexer_mode:
+        print("Will run in indexer mode")
+
+        solana_url = os.environ['SOLANA_URL']
+
+        run_indexer(solana_url)
     else:
         entry_point()

proxy/common_neon/account_whitelist.py

@@ -1,14 +1,13 @@
 import traceback
 from datetime import datetime
-import time
-from proxy.environment import ELF_PARAMS, GET_WHITE_LIST_BALANCE_MAX_RETRIES, GET_WHITE_LIST_BALANCE_RETRY_INTERVAL_S
+from proxy.environment import ELF_PARAMS
 from proxy.common_neon.permission_token import PermissionToken
 from solana.publickey import PublicKey
-from solana.rpc.api import Client as SolanaClient
 from solana.account import Account as SolanaAccount
 from typing import Union
 from proxy.common_neon.address import EthereumAddress
 from logged_groups import logged_group
+from ..common_neon.solana_interactor import SolanaInteractor
 
 NEON_MINIMAL_CLIENT_ALLOWANCE_BALANCE = int(ELF_PARAMS.get("NEON_MINIMAL_CLIENT_ALLOWANCE_BALANCE", 0))
 NEON_MINIMAL_CONTRACT_ALLOWANCE_BALANCE = int(ELF_PARAMS.get("NEON_MINIMAL_CONTRACT_ALLOWANCE_BALANCE", 0))
@@ -18,10 +17,7 @@
 
 @logged_group("neon.AccountWhitelist")
 class AccountWhitelist:
-    def __init__(self, solana: SolanaClient, payer: SolanaAccount, permission_update_int: int):
-        self.info(f'GET_WHITE_LIST_BALANCE_MAX_RETRIES={GET_WHITE_LIST_BALANCE_MAX_RETRIES}')
-        self.info(f'GET_WHITE_LIST_BALANCE_RETRY_INTERVAL_S={GET_WHITE_LIST_BALANCE_RETRY_INTERVAL_S} seconds')
-        self.info(f'permission_update_int={permission_update_int}')
+    def __init__(self, solana: SolanaInteractor, payer: SolanaAccount, permission_update_int: int):
         self.solana = solana
         self.account_cache = {}
         self.permission_update_int = permission_update_int
@@ -43,9 +39,15 @@ def __init__(self, solana: SolanaClient, payer: SolanaAccount, permission_update
                                             PublicKey(DENIAL_TOKEN_ADDR),
                                             payer)
 
-    def read_balance_diff(self, ether_addr: Union[str, EthereumAddress]):
-        allowance_balance = self.allowance_token.get_balance(ether_addr)
-        denial_balance = self.denial_token.get_balance(ether_addr)
+    def read_balance_diff(self, ether_addr: Union[str, EthereumAddress]) -> int:
+        token_list = [
+            self.allowance_token.get_token_account_address(ether_addr),
+            self.denial_token.get_token_account_address(ether_addr)
+        ]
+
+        balance_list = self.solana.get_token_account_balance_list(token_list)
+        allowance_balance = balance_list[0]
+        denial_balance = balance_list[1]
         return allowance_balance - denial_balance
 
     def grant_permissions(self, ether_addr: Union[str, EthereumAddress], min_balance: int):
@@ -106,27 +108,17 @@ def has_permission(self, ether_addr: Union[str, EthereumAddress], min_balance: i
             if diff < self.permission_update_int:
                 return cached['diff'] >= min_balance
 
-        num_retries = GET_WHITE_LIST_BALANCE_MAX_RETRIES
-
-        while True:
-            try:
-                diff = self.read_balance_diff(ether_addr)
-                self.account_cache[ether_addr] = {
-                    'last_update': current_time,
-                    'diff': diff
-                }
-                return diff >= min_balance
-            except Exception as err:
-                err_tb = "".join(traceback.format_tb(err.__traceback__))
-                self.error(f'Failed to read permissions for {ether_addr}: ' +
-                           f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}')
-                num_retries -= 1
-                if num_retries == 0:
-                    # This error should be forwarded to client
-                    raise RuntimeError('Failed to read account permissions. Try to repeat later')
-
-                self.info(f'Will retry getting whitelist balance after {GET_WHITE_LIST_BALANCE_RETRY_INTERVAL_S} seconds')
-                time.sleep(GET_WHITE_LIST_BALANCE_RETRY_INTERVAL_S)
+        try:
+            diff = self.read_balance_diff(ether_addr)
+            self.account_cache[ether_addr] = {
+                'last_update': current_time,
+                'diff': diff
+            }
+            return diff >= min_balance
+        except Exception as err:
+            err_tb = "".join(traceback.format_tb(err.__traceback__))
+            self.error(f'Failed to read permissions for {ether_addr}: ' +
+                       f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}')
 
     def has_client_permission(self, ether_addr: Union[str, EthereumAddress]):
         return self.has_permission(ether_addr, NEON_MINIMAL_CLIENT_ALLOWANCE_BALANCE)

proxy/common_neon/address.py

@@ -65,4 +65,4 @@ class AccountInfo(NamedTuple):
     @staticmethod
     def frombytes(data):
         cont = ACCOUNT_INFO_LAYOUT.parse(data)
-        return AccountInfo(cont.ether, cont.trx_count, PublicKey(cont.code_account))
+        return AccountInfo(cont.ether, int.from_bytes(cont.trx_count, 'little'), PublicKey(cont.code_account))

proxy/common_neon/costs.py

@@ -1,7 +1,7 @@
 import base58
 
 from ..environment import EVM_LOADER_ID
-from ..indexer.utils import BaseDB
+from ..indexer.base_db import BaseDB
 
 
 class SQLCost(BaseDB):

proxy/common_neon/errors.py

@@ -21,3 +21,7 @@ class SolanaAccountNotFoundError(Exception):
     """Provides special error processing"""
     def __init__(self):
         super().__init__(SolanaErrors.AccountNotFound.value)
+
+
+class PendingTxError(Exception):
+    pass

proxy/common_neon/eth_proto.py

@@ -1,6 +1,4 @@
-from ecdsa import SigningKey, SECP256k1, VerifyingKey
 from sha3 import keccak_256
-import json
 import rlp
 from eth_keys import keys
 
@@ -21,6 +19,9 @@
 '''
 
 
+class InvalidTrx(Exception):
+    pass
+
 
 class Trx(rlp.Serializable):
     fields = (
@@ -35,6 +36,9 @@ class Trx(rlp.Serializable):
         ('s', rlp.codec.big_endian_int)
     )
 
+    secpk1n = 115792089237316195423570985008687907852837564279074904382605163141518161494337
+    null_address = b'\xff' * 20
+
     def __init__(self, *args, **kwargs):
         rlp.Serializable.__init__(self, *args, **kwargs)
         self._msg = None
@@ -44,29 +48,60 @@ def fromString(cls, s):
         return rlp.decode(s, Trx)
 
     def chainId(self):
-        # chainid*2 + 35  xxxxx0 + 100011   xxxx0 + 100010 +1
-        # chainid*2 + 36  xxxxx0 + 100100   xxxx0 + 100011 +1
-        return (self.v-1)//2 - 17
+        if self.v in (27, 28):
+            return None
+        elif self.v >= 37:
+            # chainid*2 + 35  xxxxx0 + 100011   xxxx0 + 100010 +1
+            # chainid*2 + 36  xxxxx0 + 100100   xxxx0 + 100011 +1
+            return ((self.v - 1) // 2) - 17
+        else:
+            raise InvalidTrx("Invalid V value")
+
+    def _unsigned_msg(self):
+        chain_id = self.chainId()
+        if chain_id is None:
+            return rlp.encode((self.nonce, self.gasPrice, self.gasLimit, self.toAddress, self.value, self.callData))
+        else:
+            return rlp.encode((self.nonce, self.gasPrice, self.gasLimit, self.toAddress, self.value, self.callData,
+                               chain_id, 0, 0), Trx)
 
     def unsigned_msg(self):
         if self._msg is None:
-            self._msg = rlp.encode((self.nonce, self.gasPrice, self.gasLimit, self.toAddress, self.value, self.callData, self.chainId(), b"", b""))
+            self._msg = self._unsigned_msg()
         return self._msg
 
+    def _signature(self):
+        return keys.Signature(vrs=[1 if self.v % 2 == 0 else 0, self.r, self.s])
+
     def signature(self):
-        return keys.Signature(vrs=[1 if self.v % 2 == 0 else 0, self.r, self.s]).to_bytes()
+        return self._signature().to_bytes()
 
     def _sender(self):
-        hash = keccak_256(self.unsigned_msg()).digest()
-        sig = keys.Signature(vrs=[1 if self.v % 2 == 0 else 0, self.r, self.s])
-        pub = sig.recover_public_key_from_msg_hash(hash)
+        if self.r == 0 and self.s == 0:
+            return self.null_address
+        elif self.v in (27, 28):
+            pass
+        elif self.v >= 37:
+            vee = self.v - self.chainId() * 2 - 8
+            assert vee in (27, 28)
+        else:
+            raise InvalidTrx("Invalid V value")
+
+        if self.r >= self.secpk1n or self.s >= self.secpk1n or self.r == 0 or self.s == 0:
+            raise InvalidTrx("Invalid signature values!")
+
+        sighash = keccak_256(self._unsigned_msg()).digest()
+        sig = self._signature()
+        pub = sig.recover_public_key_from_msg_hash(sighash)
+
         return pub.to_canonical_address()
 
     def sender(self):
         return self._sender().hex()
 
     def hash_signed(self):
-        return keccak_256(rlp.encode((self.nonce, self.gasPrice, self.gasLimit, self.toAddress, self.value, self.callData,
+        return keccak_256(rlp.encode((self.nonce, self.gasPrice, self.gasLimit,
+                                      self.toAddress, self.value, self.callData,
                                       self.v, self.r, self.s))).digest()
 
     def contract(self):
@@ -75,6 +110,7 @@ def contract(self):
         contract_addr = rlp.encode((self._sender(), self.nonce))
         return keccak_256(contract_addr).digest()[-20:].hex()
 
+
 #class JsonEncoder(json.JSONEncoder):
 #    def default(self, obj):
 #        if isinstance(obj, bytes):

proxy/common_neon/layouts.py

@@ -1,5 +1,5 @@
 
-from construct import Bytes, Int8ul, Int64ul
+from construct import Bytes, Int8ul, Int32ul, Int64ul
 from construct import Struct
 
 STORAGE_ACCOUNT_INFO_LAYOUT = Struct(
@@ -30,6 +30,12 @@
     "ro_blocked_cnt" / Int8ul,
 )
 
+CODE_ACCOUNT_INFO_LAYOUT = Struct(
+    "type" / Int8ul,
+    "owner" / Bytes(32),
+    "code_size" / Int32ul,
+)
+
 
 CREATE_ACCOUNT_LAYOUT = Struct(
     "lamports" / Int64ul,