Shanghai hard fork support

.circleci/config.yml

@@ -135,6 +135,12 @@ jobs:
       - image: cimg/python:3.9
         environment:
           TOXENV: py39-native-blockchain-transition
+  py39-native-blockchain-shanghai:
+    <<: *common
+    docker:
+      - image: cimg/python:3.9
+        environment:
+          TOXENV: py39-native-blockchain-shanghai
 
   py39-docs:
     <<: *common
@@ -270,6 +276,7 @@ workflows:
       - py39-native-blockchain-tangerine_whistle
       - py39-native-blockchain-spurious_dragon
       - py39-native-blockchain-transition
+      - py39-native-blockchain-shanghai
       - py37-vm
       - py38-vm
       - py39-vm

eth/_utils/headers.py

@@ -1,6 +1,7 @@
 import datetime
 from typing import (
     Dict,
+    Optional,
     Tuple,
 )
 
@@ -34,7 +35,7 @@ def eth_now() -> int:
     return int(datetime.datetime.utcnow().timestamp())
 
 
-def new_timestamp_from_parent(parent: BlockHeaderAPI) -> int:
+def new_timestamp_from_parent(parent: Optional[BlockHeaderAPI]) -> int:
     """
     Generate a timestamp to use on a new header.
 

eth/abc.py

@@ -133,6 +133,16 @@ def base_fee_per_gas(self) -> Optional[int]:
         """
         ...
 
+    @property
+    @abstractmethod
+    def withdrawals_root(self) -> Optional[Hash32]:
+        """
+        Return the withdrawals root of the block.
+
+        Set to None in pre-Shanghai header.
+        """
+        ...
+
 
 class BlockHeaderSedesAPI(ABC):
     """
@@ -708,21 +718,88 @@ def encode(self) -> bytes:
         ...
 
 
+class WithdrawalAPI(ABC):
+    """
+    A class to define a withdrawal.
+    """
+
+    @property
+    @abstractmethod
+    def index(self) -> int:
+        """
+        A monotonically increasing index, starting from 0, that increments by 1 per
+        withdrawal to uniquely identify each withdrawal.
+        """
+        ...
+
+    @property
+    @abstractmethod
+    def validator_index(self) -> int:
+        """
+        The index for the validator on the consensus layer the withdrawal corresponds
+        to.
+        """
+        ...
+
+    @property
+    @abstractmethod
+    def address(self) -> Address:
+        """
+        The recipient address for the withdrawn ether.
+        """
+        ...
+
+    @property
+    @abstractmethod
+    def amount(self) -> int:
+        """
+        The nonzero amount of ether to withdraw, given in gwei (10**9 wei).
+        """
+        ...
+
+    @property
+    @abstractmethod
+    def hash(self) -> Hash32:
+        """
+        Return the hash of the withdrawal.
+        """
+        ...
+
+    @abstractmethod
+    def validate(self) -> None:
+        """
+        Validate withdrawal fields.
+        """
+        ...
+
+    @abstractmethod
+    def encode(self) -> bytes:
+        """
+        Return the encoded withdrawal.
+        """
+        ...
+
+
 class BlockAPI(ABC):
     """
     A class to define a block.
     """
     header: BlockHeaderAPI
     transactions: Tuple[SignedTransactionAPI, ...]
+    uncles: Tuple[BlockHeaderAPI, ...]
+    withdrawals: Tuple[WithdrawalAPI, ...]
+
     transaction_builder: Type[TransactionBuilderAPI] = None
     receipt_builder: Type[ReceiptBuilderAPI] = None
-    uncles: Tuple[BlockHeaderAPI, ...]
 
     @abstractmethod
-    def __init__(self,
-                 header: BlockHeaderAPI,
-                 transactions: Sequence[SignedTransactionAPI],
-                 uncles: Sequence[BlockHeaderAPI]):
+    def __init__(
+        self,
+        header: BlockHeaderAPI,
+        transactions: Sequence[SignedTransactionAPI],
+        uncles: Sequence[BlockHeaderAPI],
+        withdrawals: Sequence[WithdrawalAPI] = None,  # only present post-Shanghai
+    ) -> None:
         ...
 
     @classmethod
@@ -894,6 +971,14 @@ def make_transaction_hash_to_block_lookup_key(transaction_hash: Hash32) -> bytes
         """
         ...
 
+    @staticmethod
+    @abstractmethod
+    def make_withdrawal_hash_to_block_lookup_key(withdrawal_hash: Hash32) -> bytes:
+        """
+        Return the lookup key to retrieve a withdrawal key from a withdrawal hash.
+        """
+        ...
+
 
 class DatabaseAPI(MutableMapping[bytes, bytes], ABC):
     """
@@ -1222,6 +1307,21 @@ def get_transaction_index(self, transaction_hash: Hash32) -> Tuple[BlockNumber,
         """
         ...
 
+    #
+    # Withdrawals API
+    #
+
+    @abstractmethod
+    def get_block_withdrawals(
+        self,
+        block_header: BlockHeaderAPI,
+    ) -> Tuple[WithdrawalAPI, ...]:
+        """
+        Return an iterable of withdrawals for the block specified by the
+        given block header.
+        """
+        ...
+
     #
     # Raw Database API
     #
@@ -3040,6 +3140,15 @@ def get_transaction_context(self,
         """
         ...
 
+    #
+    # Withdrawals
+    #
+    def apply_withdrawal(self, withdrawal: WithdrawalAPI) -> None:
+        ...
+
+    def apply_all_withdrawals(self, withdrawals: Sequence[WithdrawalAPI]) -> None:
+        ...
+
 
 class ConsensusContextAPI(ABC):
     """
@@ -3252,6 +3361,15 @@ def apply_all_transactions(
         """
         ...
 
+    def apply_all_withdrawals(self, withdrawals: Sequence[WithdrawalAPI]) -> None:
+        """
+        Updates the state by applying all withdrawals.
+        This does *not* update the current block or header of the VM.
+
+        :param withdrawals: an iterable of all withdrawals to apply
+        """
+        ...
+
     @abstractmethod
     def make_receipt(self,
                      base_header: BlockHeaderAPI,
@@ -3288,13 +3406,16 @@ def mine_block(self, block: BlockAPI, *args: Any, **kwargs: Any) -> BlockAndMeta
         ...
 
     @abstractmethod
-    def set_block_transactions(self,
-                               base_block: BlockAPI,
-                               new_header: BlockHeaderAPI,
-                               transactions: Sequence[SignedTransactionAPI],
-                               receipts: Sequence[ReceiptAPI]) -> BlockAPI:
+    def set_block_transactions_and_withdrawals(
+        self,
+        base_block: BlockAPI,
+        new_header: BlockHeaderAPI,
+        transactions: Sequence[SignedTransactionAPI],
+        receipts: Sequence[ReceiptAPI],
+        withdrawals: Sequence[WithdrawalAPI] = None,
+    ) -> BlockAPI:
         """
-        Create a new block with the given ``transactions``.
+        Create a new block with the given ``transactions`` and/or ``withdrawals``.
         """
         ...
 
@@ -3882,18 +4003,20 @@ def get_canonical_block_hash(self, block_number: BlockNumber) -> Hash32:
         ...
 
     @abstractmethod
-    def build_block_with_transactions(
+    def build_block_with_transactions_and_withdrawals(
             self,
             transactions: Tuple[SignedTransactionAPI, ...],
-            parent_header: BlockHeaderAPI = None
+            parent_header: BlockHeaderAPI = None,
+            withdrawals: Tuple[WithdrawalAPI, ...] = None,
     ) -> Tuple[BlockAPI, Tuple[ReceiptAPI, ...], Tuple[ComputationAPI, ...]]:
         """
         Generate a block with the provided transactions. This does *not* import
         that block into your chain. If you want this new block in your chain,
         run :meth:`~import_block` with the result block from this method.
 
-        :param transactions: an iterable of transactions to insert to the block
+        :param transactions: an iterable of transactions to insert into the block
         :param parent_header: parent of the new block -- or canonical head if ``None``
+        :param withdrawals: an iterable of withdrawals to insert into the block
         :return: (new block, receipts, computations)
         """
         ...

eth/chains/base.py

@@ -51,6 +51,7 @@
     StateAPI,
     SignedTransactionAPI,
     UnsignedTransactionAPI,
+    WithdrawalAPI,
 )
 from eth.consensus import (
     ConsensusContext,
@@ -346,16 +347,30 @@ def get_canonical_block_by_number(self, block_number: BlockNumber) -> BlockAPI:
     def get_canonical_block_hash(self, block_number: BlockNumber) -> Hash32:
         return self.chaindb.get_canonical_block_hash(block_number)
 
-    def build_block_with_transactions(
+    def build_block_with_transactions_and_withdrawals(
             self,
             transactions: Sequence[SignedTransactionAPI],
-            parent_header: BlockHeaderAPI = None
+            parent_header: BlockHeaderAPI = None,
+            withdrawals: Sequence[WithdrawalAPI] = None,
     ) -> Tuple[BlockAPI, Tuple[ReceiptAPI, ...], Tuple[ComputationAPI, ...]]:
         base_header = self.ensure_header(parent_header)
         vm = self.get_vm(base_header)
 
-        new_header, receipts, computations = vm.apply_all_transactions(transactions, base_header)
-        new_block = vm.set_block_transactions(vm.get_block(), new_header, transactions, receipts)
+        new_header, receipts, computations = vm.apply_all_transactions(
+            transactions,
+            base_header,
+        )
+
+        if withdrawals:
+            vm.apply_all_withdrawals(withdrawals)
+
+        new_block = vm.set_block_transactions_and_withdrawals(
+            vm.get_block(),
+            new_header,
+            transactions,
+            receipts,
+            withdrawals=withdrawals,
+        )
 
         return new_block, receipts, computations
 
@@ -627,9 +642,10 @@ def __init__(self, base_db: AtomicDatabaseAPI, header: BlockHeaderAPI = None) ->
         super().__init__(base_db)
         self.header = self.ensure_header(header)
 
-    def apply_transaction(self,
-                          transaction: SignedTransactionAPI
-                          ) -> Tuple[BlockAPI, ReceiptAPI, ComputationAPI]:
+    def apply_transaction(
+        self,
+        transaction: SignedTransactionAPI,
+    ) -> Tuple[BlockAPI, ReceiptAPI, ComputationAPI]:
         vm = self.get_vm(self.header)
         base_block = vm.get_block()
 
@@ -643,7 +659,12 @@ def apply_transaction(self,
         transactions = base_block.transactions + (transaction, )
         receipts = base_block.get_receipts(self.chaindb) + (receipt, )
 
-        new_block = vm.set_block_transactions(base_block, new_header, transactions, receipts)
+        new_block = vm.set_block_transactions_and_withdrawals(
+            base_block,
+            new_header,
+            transactions,
+            receipts,
+        )
 
         self.header = new_block.header
 
@@ -673,6 +694,7 @@ def mine_all(
             transactions: Sequence[SignedTransactionAPI],
             *args: Any,
             parent_header: BlockHeaderAPI = None,
+            withdrawals: Sequence[WithdrawalAPI] = None,
             **kwargs: Any,
     ) -> Tuple[BlockImportResult, Tuple[ReceiptAPI, ...], Tuple[ComputationAPI, ...]]:
 
@@ -685,13 +707,26 @@ def mine_all(
         vm = self.get_vm(custom_header)
 
         new_header, receipts, computations = vm.apply_all_transactions(transactions, base_header)
-        filled_block = vm.set_block_transactions(vm.get_block(), new_header, transactions, receipts)
+
+        if withdrawals:
+            vm.apply_all_withdrawals(withdrawals)
+
+        filled_block = vm.set_block_transactions_and_withdrawals(
+            vm.get_block(),
+            new_header,
+            transactions,
+            receipts,
+            withdrawals=withdrawals,
+        )
 
         block_result = vm.mine_block(filled_block, *args, **kwargs)
         imported_block = block_result.block
 
         block_persist_result = self.persist_block(imported_block)
-        block_import_result = BlockImportResult(*block_persist_result, block_result.meta_witness)
+        block_import_result = BlockImportResult(
+            *block_persist_result,
+            block_result.meta_witness,
+        )
 
         self.header = self.create_header_from_parent(imported_block.header)
         return (block_import_result, receipts, computations)

eth/chains/mainnet/__init__.py

@@ -49,6 +49,7 @@
     MuirGlacierVM,
     ParisVM,
     PetersburgVM,
+    ShanghaiVM,
     SpuriousDragonVM,
     TangerineWhistleVM,
 )
@@ -126,6 +127,7 @@ class MainnetHomesteadVM(MainnetDAOValidatorVM):
 )
 POS_MAINNET_VMS = (
     ParisVM,
+    ShanghaiVM,
 )
 
 MAINNET_VMS = MINING_MAINNET_VMS + POS_MAINNET_VMS