Test test

.buildkite/steps/deploy-test.sh

@@ -74,17 +74,19 @@ echo "Wait proxy..." && wait-for-proxy "$PROXY_URL"
 
 export EVM_LOADER=$(docker exec proxy bash -c "solana address -k /spl/bin/evm_loader-keypair.json")
 export SOLANA_URL=$(docker exec solana bash -c 'echo "$SOLANA_URL"')
+export FAUCET_URL=$(docker exec proxy bash -c 'echo "$FAUCET_URL"')
 
 echo "EVM_LOADER" $EVM_LOADER
 echo "SOLANA_URL" $SOLANA_URL
+echo "FAUCET_URL" $FAUCET_URL
 
 echo "Run proxy tests..."
 docker run --rm -ti --network=container:proxy \
      -e PROXY_URL \
      -e EVM_LOADER \
      -e SOLANA_URL \
+     -e FAUCET_URL \
      -e EXTRA_GAS=100000 \
-     -e NEW_USER_AIRDROP_AMOUNT=100 \
      -e POSTGRES_DB=neon-db \
      -e POSTGRES_USER=neon-proxy \
      -e POSTGRES_PASSWORD=neon-proxy-pass \
@@ -94,7 +96,8 @@ docker run --rm -ti --network=container:proxy \
      $PROXY_IMAGE
 
 echo "Run uniswap-v2-core tests..."
-docker run --rm -ti --network=host \
+docker run --rm -ti --network=container:proxy \
+     -e FAUCET_URL \
      --entrypoint ./deploy-test.sh \
      ${EXTRA_ARGS:-} \
      $UNISWAP_V2_CORE_IMAGE \

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+proxy/docker-compose-test.yml   @vodolaz
+proxy/environment.py    @vodolaz

proxy/common_neon/neon_instruction.py

@@ -15,7 +15,7 @@
 from .address import accountWithSeed, ether2program, getTokenAddr, EthereumAddress
 from .constants import SYSVAR_INSTRUCTION_PUBKEY, INCINERATOR_PUBKEY, KECCAK_PROGRAM, COLLATERALL_POOL_MAX
 from .layouts import CREATE_ACCOUNT_LAYOUT
-from ..environment import EVM_LOADER_ID, ETH_TOKEN_MINT_ID , COLLATERAL_POOL_BASE, NEW_USER_AIRDROP_AMOUNT
+from ..environment import EVM_LOADER_ID, ETH_TOKEN_MINT_ID , COLLATERAL_POOL_BASE
 
 
 obligatory_accounts = [
@@ -186,31 +186,6 @@ def createERC20TokenAccountTrx(self, token_info) -> Transaction:
 
         return trx
 
-    def make_transfer_instruction(self, associated_token_account: PublicKey) -> TransactionInstruction:
-        transfer_instruction = transfer2(Transfer2Params(
-            source=self.operator_neon_address,
-            owner=self.operator_account,
-            dest=associated_token_account,
-            amount=NEW_USER_AIRDROP_AMOUNT * eth_utils.denoms.gwei,
-            decimals=9,
-            mint=ETH_TOKEN_MINT_ID,
-            program_id=TOKEN_PROGRAM_ID
-        ))
-        self.debug(f"Token transfer from token: {self.operator_neon_address}, owned by: {self.operator_account}, to token: "
-                    f"{associated_token_account}, owned by: {associated_token_account} , value: {NEW_USER_AIRDROP_AMOUNT}")
-        return transfer_instruction
-
-    def make_trx_with_create_and_airdrop(self, eth_account, code_acc=None) -> Transaction:
-        trx = Transaction()
-        create_trx, associated_token_account = self.make_create_eth_account_trx(eth_account, code_acc)
-        trx.add(create_trx)
-        if NEW_USER_AIRDROP_AMOUNT <= 0:
-            return trx
-        transfer_instruction = self.make_transfer_instruction(associated_token_account)
-        trx.add(transfer_instruction)
-
-        return trx
-
     def make_resize_instruction(self, account, code_account_old, code_account_new, seed) -> TransactionInstruction:
         return TransactionInstruction(
             program_id = EVM_LOADER_ID,
@@ -267,7 +242,14 @@ def make_noniterative_call_transaction(self, length_before: int = 0) -> Transact
         trx.add(self.make_05_call_instruction())
         return trx
 
-    def make_cancel_transaction(self) -> Transaction:
+    def make_cancel_transaction(self, cancel_keys = None) -> Transaction:
+        append_keys = []
+        if cancel_keys:
+            append_keys = cancel_keys
+        else:
+            append_keys = self.eth_accounts
+            append_keys.append(AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False))
+            append_keys += obligatory_accounts
         return Transaction().add(TransactionInstruction(
             program_id = EVM_LOADER_ID,
             data = bytearray.fromhex("15") + self.eth_trx.nonce.to_bytes(8, 'little'),
@@ -279,10 +261,7 @@ def make_cancel_transaction(self) -> Transaction:
                 AccountMeta(pubkey=INCINERATOR_PUBKEY, is_signer=False, is_writable=True),
                 AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False),
 
-            ] + self.eth_accounts + [
-
-                AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False),
-            ] + obligatory_accounts
+            ] + append_keys
         ))
 
     def make_partial_call_or_continue_instruction(self, steps=0) -> TransactionInstruction:

proxy/common_neon/transaction_sender.py

@@ -30,7 +30,7 @@
 from .solana_interactor import check_if_blockhash_notfound
 from ..common_neon.eth_proto import Trx as EthTx
 from ..common_neon.utils import NeonTxResultInfo, NeonTxInfo
-from ..environment import RETRY_ON_FAIL, EVM_LOADER_ID, PERM_ACCOUNT_LIMIT, ACCOUNT_PERMISSION_UPDATE_INT
+from ..environment import RETRY_ON_FAIL, EVM_LOADER_ID, PERM_ACCOUNT_LIMIT, ACCOUNT_PERMISSION_UPDATE_INT, MIN_OPERATOR_BALANCE_TO_WARN, MIN_OPERATOR_BALANCE_TO_ERR
 from ..memdb.memdb import MemDB, NeonPendingTxInfo
 from ..environment import get_solana_accounts
 from ..common_neon.account_whitelist import AccountWhitelist
@@ -119,7 +119,7 @@ def __init__(self, sender, account_desc):
 
     def _create_account(self):
         assert self.balance > 0
-        return self.s.builder.make_trx_with_create_and_airdrop(self._address)
+        return self.s.builder.make_create_eth_account_trx(self._address)[0]
 
     def build(self):
         assert self._is_empty()
@@ -168,7 +168,7 @@ def __init__(self, sender, account_desc):
 
     def _create_account(self):
         assert self.sol_account
-        return self.s.builder.make_trx_with_create_and_airdrop(self._address, self.sol_account)
+        return self.s.builder.make_create_eth_account_trx(self._address, self.sol_account)[0]
 
     def build(self):
         assert self._is_empty()
@@ -270,7 +270,7 @@ def init_resource_info(self) -> OperatorResourceInfo:
 
             with self._resource_list_len_glob.get_lock():
                 if self._resource_list_len_glob.value == 0:
-                    raise RuntimeError('No resources!')
+                    raise RuntimeError('Operator has NO resources!')
                 elif len(self._free_resource_list_glob) == 0:
                     continue
                 idx = self._free_resource_list_glob.pop(0)
@@ -289,6 +289,9 @@ def init_resource_info(self) -> OperatorResourceInfo:
         raise RuntimeError('Timeout on waiting a free operator resource!')
 
     def _init_perm_accounts(self) -> bool:
+        if self._check_operator_balance() is False:
+            self._resource_list_len_glob.value -= 1
+            return False
         if self._resource and self._resource.storage and self._resource.holder:
             return True
 
@@ -308,6 +311,24 @@ def _init_perm_accounts(self) -> bool:
             self.error(f"Fail to init accounts for resource {opkey}:{rid}, err({err}): {err_tb}")
             return False
 
+    def _min_operator_balance_to_err(self):
+        return MIN_OPERATOR_BALANCE_TO_ERR
+
+    def _min_operator_balance_to_warn(self):
+        return MIN_OPERATOR_BALANCE_TO_WARN
+
+    def _check_operator_balance(self):
+        # Validate operator's account has enough SOLs
+        sol_balance = self._s.solana.get_sol_balance(self._resource.public_key())
+        min_operator_balance_to_err = self._min_operator_balance_to_err()
+        if sol_balance <= min_operator_balance_to_err:
+            self.error(f'Operator account {self._resource.public_key()} has NOT enough SOLs; balance = {sol_balance}; min_operator_balance_to_err = {min_operator_balance_to_err}')
+            return False
+        min_operator_balance_to_warn = self._min_operator_balance_to_warn()
+        if sol_balance <= min_operator_balance_to_warn:
+            self.warning(f'Operator account {self._resource.public_key()} SOLs are running out; balance = {sol_balance}; min_operator_balance_to_warn = {min_operator_balance_to_warn}; min_operator_balance_to_err = {min_operator_balance_to_err}; ')
+        return True
+
     def _create_perm_accounts(self, seed_list):
         tx = Transaction()
 

proxy/common_neon/utils.py

@@ -159,6 +159,7 @@ def __setstate__(self, src):
         self.__dict__ = src
 
     def _set_defaults(self):
+        self.tx = None
         self.addr = None
         self.sign = None
         self.nonce = None
@@ -174,6 +175,8 @@ def _set_defaults(self):
         self.error = None
 
     def init_from_eth_tx(self, tx: EthTx):
+        self.tx = tx
+
         self.v = hex(tx.v)
         self.r = hex(tx.r)
         self.s = hex(tx.s)

proxy/docker-compose-test.yml

@@ -61,15 +61,19 @@ services:
     image: neonlabsorg/proxy:${REVISION}
     environment:
       SOLANA_URL: http://solana:8899
+      FAUCET_URL: http://faucet:3333
       POSTGRES_DB: neon-db
       POSTGRES_USER: neon-proxy
       POSTGRES_PASSWORD: neon-proxy-pass
-      NEW_USER_AIRDROP_AMOUNT: 100
       LOG_SENDING_SOLANA_TRANSACTION: "YES"
       LOG_NEON_CLI_DEBUG: "YES"
       FUZZING_BLOCKHASH: "YES"
       CONFIG: ci
-      PP_SOLANA_URL: https://api.mainnet-beta.solana.com
+      PP_SOLANA_URL: https://api.devnet.solana.com
+      PYTH_MAPPING_ACCOUNT: BmA9Z6FjioHJPpjT39QazZyhDRUdZy2ezwx4GiDdE2u2
+      MIN_OPERATOR_BALANCE_TO_WARN: 4565760000 # = 913152000 * 5 (5 storage accounts) = 4.56576 SOL
+      MIN_OPERATOR_BALANCE_TO_ERR: 913152000 # = solana rent 131072 (= Rent-exempt minimum: 0.913152 SOL) SOLs to create a storage
+      MINIMAL_GAS_PRICE: 1
     hostname: proxy
     depends_on:
       postgres:
@@ -100,9 +104,11 @@ services:
       FAUCET_SOLANA_ENABLE: 'true'
       SOLANA_URL: 'http://solana:8899'
       NEON_OPERATOR_KEYFILE: '/root/.config/solana/id.json'
-      NEON_ETH_MAX_AMOUNT: 10
-      TEST_FAUCET_INIT_NEON_BALANCE: 10000
+      NEON_ETH_MAX_AMOUNT: 1000
+      TEST_FAUCET_INIT_NEON_BALANCE: 100000
     hostname: faucet
+    ports:
+      - 127.0.0.1:3333:3333
     expose:
       - "3333"
     networks:

proxy/environment.py

@@ -13,15 +13,14 @@
 EVM_LOADER_ID = os.environ.get("EVM_LOADER")
 neon_cli_timeout = float(os.environ.get("NEON_CLI_TIMEOUT", "0.1"))
 
-NEW_USER_AIRDROP_AMOUNT = int(os.environ.get("NEW_USER_AIRDROP_AMOUNT", "0"))
 CONFIRMATION_CHECK_DELAY = float(os.environ.get("NEON_CONFIRMATION_CHECK_DELAY", "0.1"))
 CONTINUE_COUNT_FACTOR = int(os.environ.get("CONTINUE_COUNT_FACTOR", "3"))
 TIMEOUT_TO_RELOAD_NEON_CONFIG = int(os.environ.get("TIMEOUT_TO_RELOAD_NEON_CONFIG", "3600"))
 
 MINIMAL_GAS_PRICE=os.environ.get("MINIMAL_GAS_PRICE", None)
 if MINIMAL_GAS_PRICE is not None:
     MINIMAL_GAS_PRICE = int(MINIMAL_GAS_PRICE)*10**9
-
+!!!!!!!!!!!!!!!!
 EXTRA_GAS = int(os.environ.get("EXTRA_GAS", "0"))
 LOG_SENDING_SOLANA_TRANSACTION = os.environ.get("LOG_SENDING_SOLANA_TRANSACTION", "NO") == "YES"
 LOG_NEON_CLI_DEBUG = os.environ.get("LOG_NEON_CLI_DEBUG", "NO") == "YES"
@@ -45,7 +44,13 @@
 GET_SOL_PRICE_RETRY_INTERVAL = int(os.environ.get("GET_SOL_PRICE_RETRY_INTERVAL", 1))
 GET_WHITE_LIST_BALANCE_MAX_RETRIES = int(os.environ.get("GET_WHITE_LIST_BALANCE_MAX_RETRIES", 3))
 GET_WHITE_LIST_BALANCE_RETRY_INTERVAL_S = int(os.environ.get("GET_WHITE_LIST_BALANCE_RETRY_INTERVAL_S", 1))
+INDEXER_LOG_SKIP_COUNT = int(os.environ.get("INDEXER_LOG_SKIP_COUNT", 10))
+MIN_OPERATOR_BALANCE_TO_WARN = max(int(os.environ.get("MIN_OPERATOR_BALANCE_TO_WARN", 9000000000)), 9000000000)
+MIN_OPERATOR_BALANCE_TO_ERR = max(int(os.environ.get("MIN_OPERATOR_BALANCE_TO_ERR", 1000000000)), 1000000000)
 
+PYTH_MAPPING_ACCOUNT = os.environ.get("PYTH_MAPPING_ACCOUNT", None)
+if PYTH_MAPPING_ACCOUNT is not None:
+    PYTH_MAPPING_ACCOUNT = PublicKey(PYTH_MAPPING_ACCOUNT)
 
 class CliBase:
 

proxy/indexer/canceller.py

@@ -0,0 +1,73 @@
+
+import traceback
+
+from logged_groups import logged_group
+from solana.publickey import PublicKey
+from solana.rpc.api import Client
+from solana.system_program import SYS_PROGRAM_ID
+from solana.sysvar import SYSVAR_CLOCK_PUBKEY, SYSVAR_RENT_PUBKEY
+from solana.transaction import AccountMeta
+from spl.token.constants import TOKEN_PROGRAM_ID
+from spl.token.instructions import get_associated_token_address
+
+from proxy.common_neon.constants import INCINERATOR_PUBKEY, KECCAK_PROGRAM, SYSVAR_INSTRUCTION_PUBKEY
+from proxy.common_neon.neon_instruction import NeonInstruction
+from proxy.common_neon.solana_interactor import SolanaInteractor
+from proxy.common_neon.utils import get_from_dict
+from proxy.environment import ETH_TOKEN_MINT_ID, EVM_LOADER_ID, SOLANA_URL, get_solana_accounts
+
+
+@logged_group("neon.Indexer")
+class Canceller:
+    readonly_accs = [
+        PublicKey(EVM_LOADER_ID),
+        PublicKey(ETH_TOKEN_MINT_ID),
+        PublicKey(TOKEN_PROGRAM_ID),
+        PublicKey(SYSVAR_CLOCK_PUBKEY),
+        PublicKey(SYSVAR_INSTRUCTION_PUBKEY),
+        PublicKey(KECCAK_PROGRAM),
+        PublicKey(SYSVAR_RENT_PUBKEY),
+        PublicKey(INCINERATOR_PUBKEY),
+        PublicKey(SYS_PROGRAM_ID),
+    ]
+
+    def __init__(self):
+        # Initialize user account
+        self.signer = get_solana_accounts()[0]
+        self._operator = self.signer.public_key()
+        self._client = Client(SOLANA_URL)
+        self.operator_token = get_associated_token_address(PublicKey(self._operator), ETH_TOKEN_MINT_ID)
+
+        self.solana = SolanaInteractor(self._client)
+        self.builder = NeonInstruction(self._operator)
+
+
+    def unlock_accounts(self, blocked_storages):
+        for storage, tx_accounts in blocked_storages.items():
+            (neon_tx, blocked_accounts) = tx_accounts
+            if blocked_accounts is None:
+                self.error(f"Empty blocked accounts for the Neon tx {neon_tx}.")
+            else:
+                keys = []
+                for acc in blocked_accounts:
+                    is_writable = False if PublicKey(acc) in self.readonly_accs else True
+                    keys.append(AccountMeta(pubkey=acc, is_signer=False, is_writable=is_writable))
+
+                self.builder.init_eth_trx(neon_tx.tx, None, self.operator_token)
+                self.builder.init_iterative(storage, None, 0)
+
+                trx = self.builder.make_cancel_transaction(keys)
+
+                self.debug(f"Send Cancel: {trx}")
+                try:
+                    cancel_result = self.solana.send_multiple_transactions(self.signer, [trx], neon_tx.tx, "CancelWithNonce")[0]
+                    self.debug(f"cancel result: {cancel_result}")
+                    result_error = get_from_dict(cancel_result, 'meta', 'err')
+                    if result_error:
+                        self.error(f'Error sending cancel transaction: {result_error}')
+                except Exception as err:
+                    err_tb = "".join(traceback.format_tb(err.__traceback__))
+                    self.error('Exception on submitting transaction. ' +
+                               f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}')
+                else:
+                    self.debug(f"Canceled: {blocked_accounts}")