From 56ae5f8bb547c0fadfabecd45f3d69c7e36b78af Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 8 Jun 2023 11:58:17 -0700 Subject: [PATCH 01/49] feat: use mcopy for copy bytes per cancun, eip-5656 --- vyper/codegen/core.py | 31 +++++++++++++++++-------------- vyper/evm/opcodes.py | 1 + vyper/ir/optimizer.py | 6 ++++++ vyper/utils.py | 3 +-- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/vyper/codegen/core.py b/vyper/codegen/core.py index 58d9db9889..cca5e0f0a3 100644 --- a/vyper/codegen/core.py +++ b/vyper/codegen/core.py @@ -19,13 +19,8 @@ from vyper.semantics.types.shortcuts import BYTES32_T, INT256_T, UINT256_T from vyper.semantics.types.subscriptable import SArrayT from vyper.semantics.types.user import EnumT -from vyper.utils import ( - GAS_CALLDATACOPY_WORD, - GAS_CODECOPY_WORD, - GAS_IDENTITY, - GAS_IDENTITYWORD, - ceil32, -) +from vyper.utils import ( GAS_COPY_WORD, GAS_IDENTITY, GAS_IDENTITYWORD, ceil32,) +from vyper.evm.opcodes import version_check DYNAMIC_ARRAY_OVERHEAD = 1 @@ -90,12 +85,16 @@ def _identity_gas_bound(num_bytes): return GAS_IDENTITY + GAS_IDENTITYWORD * (ceil32(num_bytes) // 32) +def _mcopy_gas_bound(num_bytes): + return GAS_COPY_WORD * ceil32(num_bytes) // 32 + + def _calldatacopy_gas_bound(num_bytes): - return GAS_CALLDATACOPY_WORD * ceil32(num_bytes) // 32 + return GAS_COPY_WORD * ceil32(num_bytes) // 32 def _codecopy_gas_bound(num_bytes): - return GAS_CODECOPY_WORD * ceil32(num_bytes) // 32 + return GAS_COPY_WORD * ceil32(num_bytes) // 32 # Copy byte array word-for-word (including layout) @@ -268,8 +267,12 @@ def copy_bytes(dst, src, length, length_bound): # special cases: batch copy to memory # TODO: iloadbytes if src.location == MEMORY: - copy_op = ["staticcall", "gas", 4, src, length, dst, length] - gas_bound = _identity_gas_bound(length_bound) + if version_check(begin="cancun"): + copy_op = ["mcopy", dst, src, length] + gas_bound = _mcopy_gas_bound(length_bound) + else: + copy_op = ["staticcall", "gas", 4, src, length, dst, length] + gas_bound = _identity_gas_bound(length_bound) elif src.location == CALLDATA: copy_op = ["calldatacopy", dst, src, length] gas_bound = _calldatacopy_gas_bound(length_bound) @@ -891,11 +894,11 @@ def _complex_make_setter(left, right): assert is_tuple_like(left.typ) keys = left.typ.tuple_keys() - # if len(keyz) == 0: - # return IRnode.from_list(["pass"]) + if left.is_pointer and right.is_pointer: + len_ = left.typ.memory_bytes_required + return copy_bytes(left, right, len_, len_) # general case - # TODO use copy_bytes when the generated code is above a certain size with left.cache_when_complex("_L") as (b1, left), right.cache_when_complex("_R") as (b2, right): for k in keys: l_i = get_element_ptr(left, k, array_bounds_check=False) diff --git a/vyper/evm/opcodes.py b/vyper/evm/opcodes.py index 00e0986939..fe39dc4e28 100644 --- a/vyper/evm/opcodes.py +++ b/vyper/evm/opcodes.py @@ -88,6 +88,7 @@ "MSIZE": (0x59, 0, 1, 2), "GAS": (0x5A, 0, 1, 2), "JUMPDEST": (0x5B, 0, 0, 1), + "MCOPY": (0x5E, 3, 0, 3), "PUSH0": (0x5F, 0, 1, 2), "PUSH1": (0x60, 0, 1, 3), "PUSH2": (0x61, 0, 1, 3), diff --git a/vyper/ir/optimizer.py b/vyper/ir/optimizer.py index b13c6f79f8..c9332dd3b1 100644 --- a/vyper/ir/optimizer.py +++ b/vyper/ir/optimizer.py @@ -380,6 +380,12 @@ def _conservative_eq(x, y): # but xor is slightly easier to optimize because of being # commutative. # note that (xor (-1) x) has its own rule + # if (cond) {branch_a} + # EVM only has JUMPI, no JUMPNI + # cond iszero _join JUMPI {branch_a} _join + # (if cond already has an iszero in it, like pre-cond iszero): + # iszero iszero _join JUMPI {branch_a} _join + # iszero iszero _label JMPI => _label JUMPI return finalize("iszero", [["xor", args[0], args[1]]]) if binop == "ne" and parent_op == "iszero": diff --git a/vyper/utils.py b/vyper/utils.py index 2440117d0c..3d9d9cb416 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -196,8 +196,7 @@ def calc_mem_gas(memsize): # Specific gas usage GAS_IDENTITY = 15 GAS_IDENTITYWORD = 3 -GAS_CODECOPY_WORD = 3 -GAS_CALLDATACOPY_WORD = 3 +GAS_COPY_WORD = 3 # i.e., W_copy from YP # A decimal value can store multiples of 1/DECIMAL_DIVISOR MAX_DECIMAL_PLACES = 10 From 75ee17a6cb8bf7a571bb61af88c7ad9ba35e82bd Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 8 Jun 2023 12:19:54 -0700 Subject: [PATCH 02/49] add mcopy optimization to ir optimizer --- vyper/ir/optimizer.py | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/vyper/ir/optimizer.py b/vyper/ir/optimizer.py index c9332dd3b1..86bc9227ba 100644 --- a/vyper/ir/optimizer.py +++ b/vyper/ir/optimizer.py @@ -14,6 +14,7 @@ signed_to_unsigned, unsigned_to_signed, ) +from vyper.evm.opcodes import version_check SIGNED = False UNSIGNED = True @@ -478,6 +479,7 @@ def finalize(val, args): if value == "seq": changed |= _merge_memzero(argz) changed |= _merge_calldataload(argz) + changed |= _merge_mload(argz) changed |= _remove_empty_seqs(argz) # (seq x) => (x) for cleanliness and @@ -640,14 +642,21 @@ def _remove_empty_seqs(argz): i += 1 return changed - def _merge_calldataload(argz): - # look for sequential operations copying from calldata to memory - # and merge them into a single calldatacopy operation + return _merge_load(argz, "calldataload", "calldatacopy") + +def _merge_mload(argz): + if not version_check(begin="cancun"): + return False + return _merge_load(argz, "mload", "mcopy") + +def _merge_load(argz, _LOAD, _COPY): + # look for sequential operations copying from X to Y + # and merge them into a single copy operation changed = False mstore_nodes: List = [] - initial_mem_offset = 0 - initial_calldata_offset = 0 + initial_dst_offset = 0 + initial_src_offset = 0 total_length = 0 idx = None for i, ir_node in enumerate(argz): @@ -655,19 +664,19 @@ def _merge_calldataload(argz): if ( ir_node.value == "mstore" and isinstance(ir_node.args[0].value, int) - and ir_node.args[1].value == "calldataload" + and ir_node.args[1].value == _LOAD and isinstance(ir_node.args[1].args[0].value, int) ): # mstore of a zero value - mem_offset = ir_node.args[0].value - calldata_offset = ir_node.args[1].args[0].value + dst_offset = ir_node.args[0].value + src_offset = ir_node.args[1].args[0].value if not mstore_nodes: - initial_mem_offset = mem_offset - initial_calldata_offset = calldata_offset + initial_dst_offset = dst_offset + initial_src_offset = src_offset idx = i if ( - initial_mem_offset + total_length == mem_offset - and initial_calldata_offset + total_length == calldata_offset + initial_src_offset + total_length == src_offset + and initial_src_offset + total_length == src_offset ): mstore_nodes.append(ir_node) total_length += 32 @@ -682,7 +691,7 @@ def _merge_calldataload(argz): if len(mstore_nodes) > 1: changed = True new_ir = IRnode.from_list( - ["calldatacopy", initial_mem_offset, initial_calldata_offset, total_length], + [_COPY, initial_dst_offset, initial_src_offset, total_length], source_pos=mstore_nodes[0].source_pos, ) # replace first copy operation with optimized node and remove the rest @@ -690,8 +699,8 @@ def _merge_calldataload(argz): # note: del xs[k:l] deletes l - k items del argz[idx + 1 : idx + len(mstore_nodes)] - initial_mem_offset = 0 - initial_calldata_offset = 0 + initial_dst_offset = 0 + initial_src_offset = 0 total_length = 0 mstore_nodes.clear() From ffadff9e2e8f0ea044845ee0d0cff0dc7cd722b5 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 8 Jun 2023 12:20:20 -0700 Subject: [PATCH 03/49] fix lint --- vyper/codegen/core.py | 4 ++-- vyper/ir/optimizer.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/vyper/codegen/core.py b/vyper/codegen/core.py index cca5e0f0a3..2ed10676b7 100644 --- a/vyper/codegen/core.py +++ b/vyper/codegen/core.py @@ -1,6 +1,7 @@ from vyper import ast as vy_ast from vyper.codegen.ir_node import Encoding, IRnode from vyper.evm.address_space import CALLDATA, DATA, IMMUTABLES, MEMORY, STORAGE, TRANSIENT +from vyper.evm.opcodes import version_check from vyper.exceptions import CompilerPanic, StructureException, TypeCheckFailure, TypeMismatch from vyper.semantics.types import ( AddressT, @@ -19,8 +20,7 @@ from vyper.semantics.types.shortcuts import BYTES32_T, INT256_T, UINT256_T from vyper.semantics.types.subscriptable import SArrayT from vyper.semantics.types.user import EnumT -from vyper.utils import ( GAS_COPY_WORD, GAS_IDENTITY, GAS_IDENTITYWORD, ceil32,) -from vyper.evm.opcodes import version_check +from vyper.utils import GAS_COPY_WORD, GAS_IDENTITY, GAS_IDENTITYWORD, ceil32 DYNAMIC_ARRAY_OVERHEAD = 1 diff --git a/vyper/ir/optimizer.py b/vyper/ir/optimizer.py index 86bc9227ba..d04ed6565f 100644 --- a/vyper/ir/optimizer.py +++ b/vyper/ir/optimizer.py @@ -2,6 +2,7 @@ from typing import List, Optional, Tuple, Union from vyper.codegen.ir_node import IRnode +from vyper.evm.opcodes import version_check from vyper.exceptions import CompilerPanic, StaticAssertionException from vyper.utils import ( ceil32, @@ -14,7 +15,6 @@ signed_to_unsigned, unsigned_to_signed, ) -from vyper.evm.opcodes import version_check SIGNED = False UNSIGNED = True @@ -642,14 +642,17 @@ def _remove_empty_seqs(argz): i += 1 return changed + def _merge_calldataload(argz): return _merge_load(argz, "calldataload", "calldatacopy") + def _merge_mload(argz): if not version_check(begin="cancun"): return False return _merge_load(argz, "mload", "mcopy") + def _merge_load(argz, _LOAD, _COPY): # look for sequential operations copying from X to Y # and merge them into a single copy operation From a4d1515f2bf3cdbbeb63eb16348173bd6860a239 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 8 Jun 2023 13:18:42 -0700 Subject: [PATCH 04/49] update test_opcodes --- tests/compiler/test_opcodes.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/compiler/test_opcodes.py b/tests/compiler/test_opcodes.py index b9841b92f0..20f45ced6b 100644 --- a/tests/compiler/test_opcodes.py +++ b/tests/compiler/test_opcodes.py @@ -59,5 +59,8 @@ def test_get_opcodes(evm_version): assert "PUSH0" in ops if evm_version in ("cancun",): - assert "TLOAD" in ops - assert "TSTORE" in ops + for op in ("TLOAD", "TSTORE", "MCOPY"): + assert op in ops + else: + for op in ("TLOAD", "TSTORE", "MCOPY"): + assert op not in ops From d82970a67aa1e3cfe93899678290e4b2a67f1ebc Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 8 Jun 2023 13:18:46 -0700 Subject: [PATCH 05/49] fix versioning for MCOPY opcode --- vyper/evm/opcodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vyper/evm/opcodes.py b/vyper/evm/opcodes.py index fe39dc4e28..3c0e509511 100644 --- a/vyper/evm/opcodes.py +++ b/vyper/evm/opcodes.py @@ -88,7 +88,7 @@ "MSIZE": (0x59, 0, 1, 2), "GAS": (0x5A, 0, 1, 2), "JUMPDEST": (0x5B, 0, 0, 1), - "MCOPY": (0x5E, 3, 0, 3), + "MCOPY": (0x5E, 3, 0, (None, None, None, None, None, 3)), "PUSH0": (0x5F, 0, 1, 2), "PUSH1": (0x60, 0, 1, 3), "PUSH2": (0x61, 0, 1, 3), From 393f2a6d2909f67b06dd43afb25d337aefdc8365 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 8 Jun 2023 14:46:23 -0700 Subject: [PATCH 06/49] remove `-v` from era tester --- .github/workflows/era-tester.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/era-tester.yml b/.github/workflows/era-tester.yml index 8a2a3e50ce..187b5c03a2 100644 --- a/.github/workflows/era-tester.yml +++ b/.github/workflows/era-tester.yml @@ -101,11 +101,11 @@ jobs: if: ${{ github.ref != 'refs/heads/master' }} run: | cd era-compiler-tester - cargo run --release --bin compiler-tester -- -v --path=tests/vyper/ --mode="M0B0 ${{ env.VYPER_VERSION }}" + cargo run --release --bin compiler-tester -- --path=tests/vyper/ --mode="M0B0 ${{ env.VYPER_VERSION }}" - name: Run tester (slow) # Run era tester across the LLVM optimization matrix if: ${{ github.ref == 'refs/heads/master' }} run: | cd era-compiler-tester - cargo run --release --bin compiler-tester -- -v --path=tests/vyper/ --mode="M*B* ${{ env.VYPER_VERSION }}" + cargo run --release --bin compiler-tester -- --path=tests/vyper/ --mode="M*B* ${{ env.VYPER_VERSION }}" From 6e4b221cc15330bf8278fe44c8638f42aa4ae6a6 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 8 Jun 2023 14:50:45 -0700 Subject: [PATCH 07/49] remove dead note --- vyper/ir/optimizer.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/vyper/ir/optimizer.py b/vyper/ir/optimizer.py index d04ed6565f..31292f7279 100644 --- a/vyper/ir/optimizer.py +++ b/vyper/ir/optimizer.py @@ -381,12 +381,6 @@ def _conservative_eq(x, y): # but xor is slightly easier to optimize because of being # commutative. # note that (xor (-1) x) has its own rule - # if (cond) {branch_a} - # EVM only has JUMPI, no JUMPNI - # cond iszero _join JUMPI {branch_a} _join - # (if cond already has an iszero in it, like pre-cond iszero): - # iszero iszero _join JUMPI {branch_a} _join - # iszero iszero _label JMPI => _label JUMPI return finalize("iszero", [["xor", args[0], args[1]]]) if binop == "ne" and parent_op == "iszero": From d31a2e883e5864472373ef60dd98f06a82712d5c Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 8 Jun 2023 14:52:21 -0700 Subject: [PATCH 08/49] fix typo --- vyper/ir/optimizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vyper/ir/optimizer.py b/vyper/ir/optimizer.py index 31292f7279..8201d88b1c 100644 --- a/vyper/ir/optimizer.py +++ b/vyper/ir/optimizer.py @@ -672,7 +672,7 @@ def _merge_load(argz, _LOAD, _COPY): initial_src_offset = src_offset idx = i if ( - initial_src_offset + total_length == src_offset + initial_dst_offset + total_length == dst_offset and initial_src_offset + total_length == src_offset ): mstore_nodes.append(ir_node) From ee2a57a66bf27069b0708946ca7fb003f82e53e6 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 6 Jul 2023 16:56:12 +0000 Subject: [PATCH 09/49] fix tload/tstore availability --- vyper/evm/opcodes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vyper/evm/opcodes.py b/vyper/evm/opcodes.py index 680d403d12..27900dcb68 100644 --- a/vyper/evm/opcodes.py +++ b/vyper/evm/opcodes.py @@ -171,8 +171,8 @@ "INVALID": (0xFE, 0, 0, 0), "DEBUG": (0xA5, 1, 0, 0), "BREAKPOINT": (0xA6, 0, 0, 0), - "TLOAD": (0x5C, 1, 1, 100), - "TSTORE": (0x5D, 2, 0, 100), + "TLOAD": (0x5C, 1, 1, (None, None, None, None, None, 100)), + "TSTORE": (0x5D, 2, 0, (None, None, None, None, None, 100)), } PSEUDO_OPCODES: OpcodeMap = { From 719419c7525c1a8be2017cf08eed703a71b85075 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 6 Jul 2023 17:13:29 +0000 Subject: [PATCH 10/49] fix abi decoder --- vyper/codegen/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vyper/codegen/core.py b/vyper/codegen/core.py index 2ed10676b7..5da3515b98 100644 --- a/vyper/codegen/core.py +++ b/vyper/codegen/core.py @@ -894,7 +894,8 @@ def _complex_make_setter(left, right): assert is_tuple_like(left.typ) keys = left.typ.tuple_keys() - if left.is_pointer and right.is_pointer: + if left.is_pointer and right.is_pointer and right.encoding == Encoding.VYPER: + assert left.encoding == Encoding.VYPER len_ = left.typ.memory_bytes_required return copy_bytes(left, right, len_, len_) From 1fbe59ce10d1506a2f40adcc9231664ab204aaad Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 6 Jul 2023 17:22:06 +0000 Subject: [PATCH 11/49] add load/mstore optimization for dload --- vyper/ir/compile_ir.py | 1 + vyper/ir/optimizer.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/vyper/ir/compile_ir.py b/vyper/ir/compile_ir.py index b2a58fa8c9..1e8295c51d 100644 --- a/vyper/ir/compile_ir.py +++ b/vyper/ir/compile_ir.py @@ -296,6 +296,7 @@ def _height_of(witharg): return o # batch copy from data section of the currently executing code to memory + # (probably should have named this dcopy but oh well) elif code.value == "dloadbytes": dst = code.args[0] src = code.args[1] diff --git a/vyper/ir/optimizer.py b/vyper/ir/optimizer.py index 8201d88b1c..40e02e79c7 100644 --- a/vyper/ir/optimizer.py +++ b/vyper/ir/optimizer.py @@ -641,6 +641,10 @@ def _merge_calldataload(argz): return _merge_load(argz, "calldataload", "calldatacopy") +def _merge_dload(argz): + return _merge_load(argz, "dload", "dloadbytes") + + def _merge_mload(argz): if not version_check(begin="cancun"): return False From 8339d41990c60f6ad4df83c05da123f577598032 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 6 Jul 2023 17:40:36 +0000 Subject: [PATCH 12/49] don't always use copy_bytes --- vyper/codegen/core.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vyper/codegen/core.py b/vyper/codegen/core.py index 5da3515b98..f9ae378f0a 100644 --- a/vyper/codegen/core.py +++ b/vyper/codegen/core.py @@ -257,7 +257,6 @@ def copy_bytes(dst, src, length, length_bound): assert src.is_pointer and dst.is_pointer # fast code for common case where num bytes is small - # TODO expand this for more cases where num words is less than ~8 if length_bound <= 32: copy_op = STORE(dst, LOAD(src)) ret = IRnode.from_list(copy_op, annotation=annotation) @@ -897,7 +896,11 @@ def _complex_make_setter(left, right): if left.is_pointer and right.is_pointer and right.encoding == Encoding.VYPER: assert left.encoding == Encoding.VYPER len_ = left.typ.memory_bytes_required - return copy_bytes(left, right, len_, len_) + # 10 words is the cutoff for memory copy where identity is cheaper + # than unrolled mloads/mstores, also a good heuristic for other + # locations where we might want to start rolling the loop. + if len_ >= 32*10 or version_check(begin="cancun"): + return copy_bytes(left, right, len_, len_) # general case with left.cache_when_complex("_L") as (b1, left), right.cache_when_complex("_R") as (b2, right): From 8877e6cfb8632d15cd71b64aab9f13f42c46e459 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 6 Jul 2023 17:47:42 +0000 Subject: [PATCH 13/49] fix lint --- vyper/codegen/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vyper/codegen/core.py b/vyper/codegen/core.py index f9ae378f0a..82dedb610b 100644 --- a/vyper/codegen/core.py +++ b/vyper/codegen/core.py @@ -899,7 +899,7 @@ def _complex_make_setter(left, right): # 10 words is the cutoff for memory copy where identity is cheaper # than unrolled mloads/mstores, also a good heuristic for other # locations where we might want to start rolling the loop. - if len_ >= 32*10 or version_check(begin="cancun"): + if len_ >= 32 * 10 or version_check(begin="cancun"): return copy_bytes(left, right, len_, len_) # general case From a1b97fa18d6224975d702612e59cbd9ec27d6a87 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 6 Jul 2023 18:48:29 +0000 Subject: [PATCH 14/49] don't use copy_bytes with storage --- vyper/codegen/core.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/vyper/codegen/core.py b/vyper/codegen/core.py index 82dedb610b..2c653cf734 100644 --- a/vyper/codegen/core.py +++ b/vyper/codegen/core.py @@ -896,13 +896,28 @@ def _complex_make_setter(left, right): if left.is_pointer and right.is_pointer and right.encoding == Encoding.VYPER: assert left.encoding == Encoding.VYPER len_ = left.typ.memory_bytes_required - # 10 words is the cutoff for memory copy where identity is cheaper - # than unrolled mloads/mstores, also a good heuristic for other - # locations where we might want to start rolling the loop. - if len_ >= 32 * 10 or version_check(begin="cancun"): + + has_storage = STORAGE in (left.location, right.location) + if has_storage: + # TODO: make this smarter, probably want to even loop for storage + # above a certain threshold. note a single sstore(dst (sload src)) + # is 8 bytes, whereas loop overhead is 17 bytes. + should_batch_copy = False + else: + # 10 words is the cutoff for memory copy where identity is cheaper + # than unrolled mloads/mstores + # if MCOPY is available, mcopy is *always* better (except in + # the 1 word case, but that is already handled by copy_bytes). + if right.location == MEMORY: + should_batch_copy = (len_ >= 32 * 10 or version_check(begin="cancun")) + # calldata or code to memory - batch copy is always better. + else: + should_batch_copy = True + + if should_batch_copy: return copy_bytes(left, right, len_, len_) - # general case + # general case, unroll with left.cache_when_complex("_L") as (b1, left), right.cache_when_complex("_R") as (b2, right): for k in keys: l_i = get_element_ptr(left, k, array_bounds_check=False) From a98871251a1be65f19296ebb30aa7709c4cb012a Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 6 Jul 2023 18:49:11 +0000 Subject: [PATCH 15/49] fix lint --- vyper/codegen/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vyper/codegen/core.py b/vyper/codegen/core.py index 2c653cf734..a330059ec7 100644 --- a/vyper/codegen/core.py +++ b/vyper/codegen/core.py @@ -909,7 +909,7 @@ def _complex_make_setter(left, right): # if MCOPY is available, mcopy is *always* better (except in # the 1 word case, but that is already handled by copy_bytes). if right.location == MEMORY: - should_batch_copy = (len_ >= 32 * 10 or version_check(begin="cancun")) + should_batch_copy = len_ >= 32 * 10 or version_check(begin="cancun") # calldata or code to memory - batch copy is always better. else: should_batch_copy = True From 8ed424b77966f8fa352ec4b737221aa543746b6f Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 6 Jul 2023 21:17:33 -0400 Subject: [PATCH 16/49] feat: add optimization flag to vyper compiler this commit adds the `--optimize` flag to the vyper cli, and as an option in vyper json. it is to be used separately from the `--no-optimize` flag. this commit does not actually add different options, just adds the flag and threads it through the codebase so it is available once we want to start differentiating between the two modes, and sets up the test harness to test both modes. --- .github/workflows/test.yml | 4 ++-- docs/compiling-a-contract.rst | 7 ++++--- tests/base_conftest.py | 16 ++++++++-------- tests/conftest.py | 25 ++++++++++++++----------- tox.ini | 1 + vyper/cli/vyper_compile.py | 16 ++++++++++------ vyper/cli/vyper_json.py | 14 ++++++++++++-- vyper/codegen/global_context.py | 3 ++- vyper/compiler/__init__.py | 16 ++++++++-------- vyper/compiler/phases.py | 22 +++++++++++----------- vyper/compiler/settings.py | 21 +++++++++++++++++++++ vyper/ir/compile_ir.py | 4 ++-- 12 files changed, 95 insertions(+), 54 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 42e0524b13..36ad477d0e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,8 +79,8 @@ jobs: strategy: matrix: python-version: [["3.10", "310"], ["3.11", "311"]] - # run in default (optimized) and --no-optimize mode - flag: ["core", "no-opt"] + # run in default (optimized, gas) and --no-optimize mode + flag: ["core", "no-opt", "codesize"] name: py${{ matrix.python-version[1] }}-${{ matrix.flag }} diff --git a/docs/compiling-a-contract.rst b/docs/compiling-a-contract.rst index 6295226bca..bb5c4db3d0 100644 --- a/docs/compiling-a-contract.rst +++ b/docs/compiling-a-contract.rst @@ -213,9 +213,10 @@ The following example describes the expected input format of ``vyper-json``. Com // Optional "settings": { "evmVersion": "shanghai", // EVM version to compile for. Can be istanbul, berlin, paris, shanghai (default) or cancun (experimental!). - // optional, whether or not optimizations are turned on - // defaults to true - "optimize": true, + // optional, optimization mode + // defaults to "gas". can be one of "gas", "codesize", "none", + // false and true (the last two are for backwards compatibility). + "optimize": "gas", // optional, whether or not the bytecode should include Vyper's signature // defaults to true "bytecodeMetadata": true, diff --git a/tests/base_conftest.py b/tests/base_conftest.py index 29809a074d..e509957e46 100644 --- a/tests/base_conftest.py +++ b/tests/base_conftest.py @@ -111,13 +111,13 @@ def w3(tester): return w3 -def _get_contract(w3, source_code, no_optimize, *args, **kwargs): +def _get_contract(w3, source_code, optimize, *args, **kwargs): out = compiler.compile_code( source_code, # test that metadata gets generated ["abi", "bytecode", "metadata"], interface_codes=kwargs.pop("interface_codes", None), - no_optimize=no_optimize, + optimize=optimize, evm_version=kwargs.pop("evm_version", None), show_gas_estimates=True, # Enable gas estimates for testing ) @@ -135,12 +135,12 @@ def _get_contract(w3, source_code, no_optimize, *args, **kwargs): return w3.eth.contract(address, abi=abi, bytecode=bytecode, ContractFactoryClass=VyperContract) -def _deploy_blueprint_for(w3, source_code, no_optimize, initcode_prefix=b"", **kwargs): +def _deploy_blueprint_for(w3, source_code, optimize, initcode_prefix=b"", **kwargs): out = compiler.compile_code( source_code, ["abi", "bytecode"], interface_codes=kwargs.pop("interface_codes", None), - no_optimize=no_optimize, + optimize=optimize, evm_version=kwargs.pop("evm_version", None), show_gas_estimates=True, # Enable gas estimates for testing ) @@ -173,17 +173,17 @@ def factory(address): @pytest.fixture(scope="module") -def deploy_blueprint_for(w3, no_optimize): +def deploy_blueprint_for(w3, optimize): def deploy_blueprint_for(source_code, *args, **kwargs): - return _deploy_blueprint_for(w3, source_code, no_optimize, *args, **kwargs) + return _deploy_blueprint_for(w3, source_code, optimize, *args, **kwargs) return deploy_blueprint_for @pytest.fixture(scope="module") -def get_contract(w3, no_optimize): +def get_contract(w3, optimize): def get_contract(source_code, *args, **kwargs): - return _get_contract(w3, source_code, no_optimize, *args, **kwargs) + return _get_contract(w3, source_code, optimize, *args, **kwargs) return get_contract diff --git a/tests/conftest.py b/tests/conftest.py index 1cc9e4e72e..24825f9dc4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -40,8 +40,11 @@ def pytest_addoption(parser): @pytest.fixture(scope="module") -def no_optimize(pytestconfig): - return pytestconfig.getoption("no_optimize") +def optimize(pytestconfig): + flag = pytestconfig.getoption("optimize") + if not flag: + return OptimizationLevel.GAS + return OptimizationLevel.from_string(flag) @pytest.fixture @@ -58,13 +61,13 @@ def bytes_helper(str, length): @pytest.fixture -def get_contract_from_ir(w3, no_optimize): +def get_contract_from_ir(w3, optimize): def ir_compiler(ir, *args, **kwargs): ir = IRnode.from_list(ir) - if not no_optimize: + if optimize != OptimizationLevel.NONE: ir = optimizer.optimize(ir) bytecode, _ = compile_ir.assembly_to_evm( - compile_ir.compile_to_assembly(ir, no_optimize=no_optimize) + compile_ir.compile_to_assembly(ir, optimize=optimize) ) abi = kwargs.get("abi") or [] c = w3.eth.contract(abi=abi, bytecode=bytecode) @@ -80,7 +83,7 @@ def ir_compiler(ir, *args, **kwargs): @pytest.fixture(scope="module") -def get_contract_module(no_optimize): +def get_contract_module(optimize): """ This fixture is used for Hypothesis tests to ensure that the same contract is called over multiple runs of the test. @@ -93,7 +96,7 @@ def get_contract_module(no_optimize): w3.eth.set_gas_price_strategy(zero_gas_price_strategy) def get_contract_module(source_code, *args, **kwargs): - return _get_contract(w3, source_code, no_optimize, *args, **kwargs) + return _get_contract(w3, source_code, optimize, *args, **kwargs) return get_contract_module @@ -138,9 +141,9 @@ def set_decorator_to_contract_function(w3, tester, contract, source_code, func): @pytest.fixture -def get_contract_with_gas_estimation(tester, w3, no_optimize): +def get_contract_with_gas_estimation(tester, w3, optimize): def get_contract_with_gas_estimation(source_code, *args, **kwargs): - contract = _get_contract(w3, source_code, no_optimize, *args, **kwargs) + contract = _get_contract(w3, source_code, optimize, *args, **kwargs) for abi_ in contract._classic_contract.functions.abi: if abi_["type"] == "function": set_decorator_to_contract_function(w3, tester, contract, source_code, abi_["name"]) @@ -150,9 +153,9 @@ def get_contract_with_gas_estimation(source_code, *args, **kwargs): @pytest.fixture -def get_contract_with_gas_estimation_for_constants(w3, no_optimize): +def get_contract_with_gas_estimation_for_constants(w3, optimize): def get_contract_with_gas_estimation_for_constants(source_code, *args, **kwargs): - return _get_contract(w3, source_code, no_optimize, *args, **kwargs) + return _get_contract(w3, source_code, optimize, *args, **kwargs) return get_contract_with_gas_estimation_for_constants diff --git a/tox.ini b/tox.ini index 5ddd01d7d4..81ff7a3397 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,7 @@ usedevelop = True commands = core: pytest -m "not fuzzing" --showlocals {posargs:tests/} no-opt: pytest -m "not fuzzing" --showlocals --no-optimize {posargs:tests/} + codesize: pytest -m "not fuzzing" --showlocals --optimize codesize {posargs:tests/} basepython = py310: python3.10 py311: python3.11 diff --git a/vyper/cli/vyper_compile.py b/vyper/cli/vyper_compile.py index f5e113116d..f279e90139 100755 --- a/vyper/cli/vyper_compile.py +++ b/vyper/cli/vyper_compile.py @@ -11,7 +11,7 @@ import vyper.codegen.ir_node as ir_node from vyper.cli import vyper_json from vyper.cli.utils import extract_file_interface_imports, get_interface_file_path -from vyper.compiler.settings import VYPER_TRACEBACK_LIMIT +from vyper.compiler.settings import VYPER_TRACEBACK_LIMIT, OptimizationLevel from vyper.evm.opcodes import DEFAULT_EVM_VERSION, EVM_VERSIONS from vyper.typing import ContractCodes, ContractPath, OutputFormats @@ -37,8 +37,6 @@ ir - Intermediate representation in list format ir_json - Intermediate representation in JSON format hex-ir - Output IR and assembly constants in hex instead of decimal -no-optimize - Do not optimize (don't use this for production code) -no-bytecode-metadata - Do not add metadata to bytecode """ combined_json_outputs = [ @@ -108,6 +106,7 @@ def _parse_args(argv): dest="evm_version", ) parser.add_argument("--no-optimize", help="Do not optimize", action="store_true") + parser.add_argument("--optimize", help="Optimization flag", choices=["gas", "codesize"]) parser.add_argument( "--no-bytecode-metadata", help="Do not add metadata to bytecode", action="store_true" ) @@ -153,13 +152,18 @@ def _parse_args(argv): output_formats = tuple(uniq(args.format.split(","))) + if args.no_optimize and args.optimize: + raise ValueError("Cannot use `--no-optimize` and `--optimize` at the same time!") + + optimize = OptimizationLevel.NONE if args.no_optimize else OptimizationLevel.from_string(args.optimize) + compiled = compile_files( args.input_files, output_formats, args.root_folder, args.show_gas_estimates, args.evm_version, - args.no_optimize, + optimize, args.storage_layout, args.no_bytecode_metadata, ) @@ -254,7 +258,7 @@ def compile_files( root_folder: str = ".", show_gas_estimates: bool = False, evm_version: str = DEFAULT_EVM_VERSION, - no_optimize: bool = False, + optimize: OptimizationLevel = OptimizationLevel.GAS, storage_layout: Iterable[str] = None, no_bytecode_metadata: bool = False, ) -> OrderedDict: @@ -297,7 +301,7 @@ def compile_files( exc_handler=exc_handler, interface_codes=get_interface_codes(root_path, contract_sources), evm_version=evm_version, - no_optimize=no_optimize, + optimize=optimize, storage_layouts=storage_layouts, show_gas_estimates=show_gas_estimates, no_bytecode_metadata=no_bytecode_metadata, diff --git a/vyper/cli/vyper_json.py b/vyper/cli/vyper_json.py index 9778848aa2..0012f88723 100755 --- a/vyper/cli/vyper_json.py +++ b/vyper/cli/vyper_json.py @@ -9,6 +9,7 @@ import vyper from vyper.cli.utils import extract_file_interface_imports, get_interface_file_path +from vyper.compiler.settings import OptimizationLevel from vyper.evm.opcodes import DEFAULT_EVM_VERSION, EVM_VERSIONS from vyper.exceptions import JSONError from vyper.typing import ContractCodes, ContractPath @@ -360,7 +361,16 @@ def compile_from_input_dict( raise JSONError(f"Invalid language '{input_dict['language']}' - Only Vyper is supported.") evm_version = get_evm_version(input_dict) - no_optimize = not input_dict["settings"].get("optimize", True) + input_dict["settings"].get() + + optimize = input_dict["settings"].get("optimize", "gas") + if isinstance(optimize, bool): + # bool optimization level for backwards compatibility + warnings.warn("optimize: is deprecated! please use one of 'gas', 'codesize', 'none'.") + optimize = OptimizationLevel.GAS if optimize else OptimizationLevel.NONE + else: + optimize = OptimizationLevel.from_string(optimize) + no_bytecode_metadata = not input_dict["settings"].get("bytecodeMetadata", True) contract_sources: ContractCodes = get_input_dict_contracts(input_dict) @@ -383,7 +393,7 @@ def compile_from_input_dict( output_formats[contract_path], interface_codes=interface_codes, initial_id=id_, - no_optimize=no_optimize, + optimize=optimize, evm_version=evm_version, no_bytecode_metadata=no_bytecode_metadata, ) diff --git a/vyper/codegen/global_context.py b/vyper/codegen/global_context.py index 1f6783f6f8..19c63e62e3 100644 --- a/vyper/codegen/global_context.py +++ b/vyper/codegen/global_context.py @@ -2,12 +2,13 @@ from typing import Optional from vyper import ast as vy_ast +from vyper.compiler.settings import OptimizationLevel # Datatype to store all global context information. # TODO: rename me to ModuleT class GlobalContext: - def __init__(self, module: Optional[vy_ast.Module] = None): + def __init__(self, module: Optional[vy_ast.Module] = None, optimize = OptimizationLevel.GAS): self._module = module @cached_property diff --git a/vyper/compiler/__init__.py b/vyper/compiler/__init__.py index 7be45ce832..1c586bd48a 100644 --- a/vyper/compiler/__init__.py +++ b/vyper/compiler/__init__.py @@ -53,7 +53,7 @@ def compile_codes( exc_handler: Union[Callable, None] = None, interface_codes: Union[InterfaceDict, InterfaceImports, None] = None, initial_id: int = 0, - no_optimize: bool = False, + optimize: OptimizationLevel = OptimizationLevel.GAS, storage_layouts: Dict[ContractPath, StorageLayout] = None, show_gas_estimates: bool = False, no_bytecode_metadata: bool = False, @@ -76,8 +76,8 @@ def compile_codes( evm_version: str, optional The target EVM ruleset to compile for. If not given, defaults to the latest implemented ruleset. - no_optimize: bool, optional - Turn off optimizations. Defaults to False + optimize: OptimizerLevel, optional + Set optimization mode. Defaults to OptimizationLevel.GAS show_gas_estimates: bool, optional Show gas estimates for abi and ir output modes interface_codes: Dict, optional @@ -126,7 +126,7 @@ def compile_codes( contract_name, interfaces, source_id, - no_optimize, + optimize, storage_layout_override, show_gas_estimates, no_bytecode_metadata, @@ -154,7 +154,7 @@ def compile_code( output_formats: Optional[OutputFormats] = None, interface_codes: Optional[InterfaceImports] = None, evm_version: str = DEFAULT_EVM_VERSION, - no_optimize: bool = False, + optimize: OptimizerLevel = OptimizerLevel.GAS, storage_layout_override: StorageLayout = None, show_gas_estimates: bool = False, ) -> dict: @@ -171,8 +171,8 @@ def compile_code( evm_version: str, optional The target EVM ruleset to compile for. If not given, defaults to the latest implemented ruleset. - no_optimize: bool, optional - Turn off optimizations. Defaults to False + optimize: OptimizerLevel, optional + Set optimization mode. Defaults to OptimizationLevel.GAS show_gas_estimates: bool, optional Show gas estimates for abi and ir output modes interface_codes: Dict, optional @@ -195,7 +195,7 @@ def compile_code( output_formats, interface_codes=interface_codes, evm_version=evm_version, - no_optimize=no_optimize, + optimize=optimize, storage_layouts=storage_layouts, show_gas_estimates=show_gas_estimates, )[UNKNOWN_CONTRACT_NAME] diff --git a/vyper/compiler/phases.py b/vyper/compiler/phases.py index c759f6e272..5c492c845b 100644 --- a/vyper/compiler/phases.py +++ b/vyper/compiler/phases.py @@ -49,7 +49,7 @@ def __init__( contract_name: str = "VyperContract", interface_codes: Optional[InterfaceImports] = None, source_id: int = 0, - no_optimize: bool = False, + optimize: OptimizationLevel = OptimizationLevel.GAS, storage_layout: StorageLayout = None, show_gas_estimates: bool = False, no_bytecode_metadata: bool = False, @@ -69,8 +69,8 @@ def __init__( * JSON interfaces are given as lists, vyper interfaces as strings source_id : int, optional ID number used to identify this contract in the source map. - no_optimize: bool, optional - Turn off optimizations. Defaults to False + optimize: OptimizationLevel, optional + Set optimization mode. Defaults to OptimizationLevel.GAS show_gas_estimates: bool, optional Show gas estimates for abi and ir output modes no_bytecode_metadata: bool, optional @@ -80,7 +80,7 @@ def __init__( self.source_code = source_code self.interface_codes = interface_codes self.source_id = source_id - self.no_optimize = no_optimize + self.optimize = optimize self.storage_layout_override = storage_layout self.show_gas_estimates = show_gas_estimates self.no_bytecode_metadata = no_bytecode_metadata @@ -119,7 +119,7 @@ def global_ctx(self) -> GlobalContext: @cached_property def _ir_output(self): # fetch both deployment and runtime IR - return generate_ir_nodes(self.global_ctx, self.no_optimize) + return generate_ir_nodes(self.global_ctx, self.optimize) @property def ir_nodes(self) -> IRnode: @@ -142,11 +142,11 @@ def function_signatures(self) -> dict[str, ContractFunctionT]: @cached_property def assembly(self) -> list: - return generate_assembly(self.ir_nodes, self.no_optimize) + return generate_assembly(self.ir_nodes, self.optimize) @cached_property def assembly_runtime(self) -> list: - return generate_assembly(self.ir_runtime, self.no_optimize) + return generate_assembly(self.ir_runtime, self.optimize) @cached_property def bytecode(self) -> bytes: @@ -233,7 +233,7 @@ def generate_folded_ast( return vyper_module_folded, symbol_tables -def generate_ir_nodes(global_ctx: GlobalContext, no_optimize: bool) -> tuple[IRnode, IRnode]: +def generate_ir_nodes(global_ctx: GlobalContext, optimize: bool) -> tuple[IRnode, IRnode]: """ Generate the intermediate representation (IR) from the contextualized AST. @@ -254,13 +254,13 @@ def generate_ir_nodes(global_ctx: GlobalContext, no_optimize: bool) -> tuple[IRn IR to generate runtime bytecode """ ir_nodes, ir_runtime = module.generate_ir_for_module(global_ctx) - if not no_optimize: + if optimize != OptimizationLevel.NONE: ir_nodes = optimizer.optimize(ir_nodes) ir_runtime = optimizer.optimize(ir_runtime) return ir_nodes, ir_runtime -def generate_assembly(ir_nodes: IRnode, no_optimize: bool = False) -> list: +def generate_assembly(ir_nodes: IRnode, optimize: OptimizationLevel = OptimizationLevel.GAS) -> list: """ Generate assembly instructions from IR. @@ -274,7 +274,7 @@ def generate_assembly(ir_nodes: IRnode, no_optimize: bool = False) -> list: list List of assembly instructions. """ - assembly = compile_ir.compile_to_assembly(ir_nodes, no_optimize=no_optimize) + assembly = compile_ir.compile_to_assembly(ir_nodes, optimize=optimize) if _find_nested_opcode(assembly, "DEBUG"): warnings.warn( diff --git a/vyper/compiler/settings.py b/vyper/compiler/settings.py index 09ced0dcb8..344e3366d9 100644 --- a/vyper/compiler/settings.py +++ b/vyper/compiler/settings.py @@ -1,5 +1,6 @@ import os from typing import Optional +from enum import Enum VYPER_COLOR_OUTPUT = os.environ.get("VYPER_COLOR_OUTPUT", "0") == "1" VYPER_ERROR_CONTEXT_LINES = int(os.environ.get("VYPER_ERROR_CONTEXT_LINES", "1")) @@ -12,3 +13,23 @@ VYPER_TRACEBACK_LIMIT = int(_tb_limit_str) else: VYPER_TRACEBACK_LIMIT = None + +class OptimizationLevel(Enum): + NONE = 0 + GAS = 1 + CODESIZE = 2 + + @classmethod + def from_string(cls, val): + match val: + case "none": + return cls.NONE + case "gas": + return cls.GAS + case "codesize": + return cls.CODESIZE + raise ValueError(f"unrecognized optimization level: {val}") + + #@classmethod + #def default(cls): + # return cls.GAS diff --git a/vyper/ir/compile_ir.py b/vyper/ir/compile_ir.py index 5a35b8f932..a8e24366c4 100644 --- a/vyper/ir/compile_ir.py +++ b/vyper/ir/compile_ir.py @@ -201,7 +201,7 @@ def apply_line_no_wrapper(*args, **kwargs): @apply_line_numbers -def compile_to_assembly(code, no_optimize=False): +def compile_to_assembly(code, optimize=OptimizationLevel.GAS): global _revert_label _revert_label = mksymbol("revert") @@ -212,7 +212,7 @@ def compile_to_assembly(code, no_optimize=False): res = _compile_to_assembly(code) _add_postambles(res) - if not no_optimize: + if optimize != OptimizationLevel.NONE: _optimize_assembly(res) return res From 017a19f3c69af1c521bae7edd24761aae379e919 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 7 Jul 2023 01:34:02 +0000 Subject: [PATCH 17/49] fix lint --- tests/conftest.py | 1 + vyper/cli/vyper_compile.py | 4 +++- vyper/cli/vyper_json.py | 4 +++- vyper/codegen/global_context.py | 2 +- vyper/compiler/__init__.py | 3 ++- vyper/compiler/phases.py | 5 ++++- vyper/compiler/settings.py | 7 ++++--- vyper/ir/compile_ir.py | 1 + 8 files changed, 19 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 24825f9dc4..3a37609c24 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,7 @@ from vyper import compiler from vyper.codegen.ir_node import IRnode +from vyper.compiler.esttings import OptimizationLevel from vyper.ir import compile_ir, optimizer from .base_conftest import VyperContract, _get_contract, zero_gas_price_strategy diff --git a/vyper/cli/vyper_compile.py b/vyper/cli/vyper_compile.py index f279e90139..0ce513fc5e 100755 --- a/vyper/cli/vyper_compile.py +++ b/vyper/cli/vyper_compile.py @@ -155,7 +155,9 @@ def _parse_args(argv): if args.no_optimize and args.optimize: raise ValueError("Cannot use `--no-optimize` and `--optimize` at the same time!") - optimize = OptimizationLevel.NONE if args.no_optimize else OptimizationLevel.from_string(args.optimize) + optimize = ( + OptimizationLevel.NONE if args.no_optimize else OptimizationLevel.from_string(args.optimize) + ) compiled = compile_files( args.input_files, diff --git a/vyper/cli/vyper_json.py b/vyper/cli/vyper_json.py index 0012f88723..5a1e7c71fa 100755 --- a/vyper/cli/vyper_json.py +++ b/vyper/cli/vyper_json.py @@ -366,7 +366,9 @@ def compile_from_input_dict( optimize = input_dict["settings"].get("optimize", "gas") if isinstance(optimize, bool): # bool optimization level for backwards compatibility - warnings.warn("optimize: is deprecated! please use one of 'gas', 'codesize', 'none'.") + warnings.warn( + "optimize: is deprecated! please use one of 'gas', 'codesize', 'none'." + ) optimize = OptimizationLevel.GAS if optimize else OptimizationLevel.NONE else: optimize = OptimizationLevel.from_string(optimize) diff --git a/vyper/codegen/global_context.py b/vyper/codegen/global_context.py index 19c63e62e3..613c1a1d2f 100644 --- a/vyper/codegen/global_context.py +++ b/vyper/codegen/global_context.py @@ -8,7 +8,7 @@ # Datatype to store all global context information. # TODO: rename me to ModuleT class GlobalContext: - def __init__(self, module: Optional[vy_ast.Module] = None, optimize = OptimizationLevel.GAS): + def __init__(self, module: Optional[vy_ast.Module] = None, optimize=OptimizationLevel.GAS): self._module = module @cached_property diff --git a/vyper/compiler/__init__.py b/vyper/compiler/__init__.py index 1c586bd48a..9affabc0ac 100644 --- a/vyper/compiler/__init__.py +++ b/vyper/compiler/__init__.py @@ -5,6 +5,7 @@ import vyper.codegen.core as codegen import vyper.compiler.output as output from vyper.compiler.phases import CompilerData +from vyper.compiler.settings import OptimizerLevel from vyper.evm.opcodes import DEFAULT_EVM_VERSION, evm_wrapper from vyper.typing import ( ContractCodes, @@ -53,7 +54,7 @@ def compile_codes( exc_handler: Union[Callable, None] = None, interface_codes: Union[InterfaceDict, InterfaceImports, None] = None, initial_id: int = 0, - optimize: OptimizationLevel = OptimizationLevel.GAS, + optimize: OptimizerLevel = OptimizerLevel.GAS, storage_layouts: Dict[ContractPath, StorageLayout] = None, show_gas_estimates: bool = False, no_bytecode_metadata: bool = False, diff --git a/vyper/compiler/phases.py b/vyper/compiler/phases.py index 5c492c845b..1a9e65f88c 100644 --- a/vyper/compiler/phases.py +++ b/vyper/compiler/phases.py @@ -7,6 +7,7 @@ from vyper.codegen import module from vyper.codegen.global_context import GlobalContext from vyper.codegen.ir_node import IRnode +from vyper.compiler.settings import OptimizationLevel from vyper.ir import compile_ir, optimizer from vyper.semantics import set_data_positions, validate_semantics from vyper.semantics.types.function import ContractFunctionT @@ -260,7 +261,9 @@ def generate_ir_nodes(global_ctx: GlobalContext, optimize: bool) -> tuple[IRnode return ir_nodes, ir_runtime -def generate_assembly(ir_nodes: IRnode, optimize: OptimizationLevel = OptimizationLevel.GAS) -> list: +def generate_assembly( + ir_nodes: IRnode, optimize: OptimizationLevel = OptimizationLevel.GAS +) -> list: """ Generate assembly instructions from IR. diff --git a/vyper/compiler/settings.py b/vyper/compiler/settings.py index 344e3366d9..1647bb5691 100644 --- a/vyper/compiler/settings.py +++ b/vyper/compiler/settings.py @@ -1,6 +1,6 @@ import os -from typing import Optional from enum import Enum +from typing import Optional VYPER_COLOR_OUTPUT = os.environ.get("VYPER_COLOR_OUTPUT", "0") == "1" VYPER_ERROR_CONTEXT_LINES = int(os.environ.get("VYPER_ERROR_CONTEXT_LINES", "1")) @@ -14,6 +14,7 @@ else: VYPER_TRACEBACK_LIMIT = None + class OptimizationLevel(Enum): NONE = 0 GAS = 1 @@ -30,6 +31,6 @@ def from_string(cls, val): return cls.CODESIZE raise ValueError(f"unrecognized optimization level: {val}") - #@classmethod - #def default(cls): + # @classmethod + # def default(cls): # return cls.GAS diff --git a/vyper/ir/compile_ir.py b/vyper/ir/compile_ir.py index a8e24366c4..15a68a5079 100644 --- a/vyper/ir/compile_ir.py +++ b/vyper/ir/compile_ir.py @@ -3,6 +3,7 @@ import math from vyper.codegen.ir_node import IRnode +from vyper.compiler.settings import OptimizationLevel from vyper.evm.opcodes import get_opcodes, version_check from vyper.exceptions import CodegenPanic, CompilerPanic from vyper.utils import MemoryPositions From 37d7e644261daf7e3ab3ea543845c226dee61a93 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 7 Jul 2023 01:42:16 +0000 Subject: [PATCH 18/49] fix typo --- tests/conftest.py | 6 ++---- vyper/cli/vyper_compile.py | 7 ++++--- vyper/cli/vyper_json.py | 1 - vyper/compiler/__init__.py | 10 +++++----- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3a37609c24..015ccae177 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,7 +10,7 @@ from vyper import compiler from vyper.codegen.ir_node import IRnode -from vyper.compiler.esttings import OptimizationLevel +from vyper.compiler.settings import OptimizationLevel from vyper.ir import compile_ir, optimizer from .base_conftest import VyperContract, _get_contract, zero_gas_price_strategy @@ -37,14 +37,12 @@ def set_evm_verbose_logging(): def pytest_addoption(parser): - parser.addoption("--no-optimize", action="store_true", help="disable asm and IR optimizations") + parser.addoption("--optimize", choices=["codesize", "gas", "none"], default="gas", help="disable asm and IR optimizations") @pytest.fixture(scope="module") def optimize(pytestconfig): flag = pytestconfig.getoption("optimize") - if not flag: - return OptimizationLevel.GAS return OptimizationLevel.from_string(flag) diff --git a/vyper/cli/vyper_compile.py b/vyper/cli/vyper_compile.py index 0ce513fc5e..2af2ce6493 100755 --- a/vyper/cli/vyper_compile.py +++ b/vyper/cli/vyper_compile.py @@ -155,9 +155,10 @@ def _parse_args(argv): if args.no_optimize and args.optimize: raise ValueError("Cannot use `--no-optimize` and `--optimize` at the same time!") - optimize = ( - OptimizationLevel.NONE if args.no_optimize else OptimizationLevel.from_string(args.optimize) - ) + if args.no_optimize or not args.optimize: + optimize = OptimizationLevel.NONE + else: + optimize = OptimizationLevel.from_string(args.optimize) compiled = compile_files( args.input_files, diff --git a/vyper/cli/vyper_json.py b/vyper/cli/vyper_json.py index 5a1e7c71fa..bc71bc3fd0 100755 --- a/vyper/cli/vyper_json.py +++ b/vyper/cli/vyper_json.py @@ -361,7 +361,6 @@ def compile_from_input_dict( raise JSONError(f"Invalid language '{input_dict['language']}' - Only Vyper is supported.") evm_version = get_evm_version(input_dict) - input_dict["settings"].get() optimize = input_dict["settings"].get("optimize", "gas") if isinstance(optimize, bool): diff --git a/vyper/compiler/__init__.py b/vyper/compiler/__init__.py index 9affabc0ac..fae0641ecf 100644 --- a/vyper/compiler/__init__.py +++ b/vyper/compiler/__init__.py @@ -5,7 +5,7 @@ import vyper.codegen.core as codegen import vyper.compiler.output as output from vyper.compiler.phases import CompilerData -from vyper.compiler.settings import OptimizerLevel +from vyper.compiler.settings import OptimizationLevel from vyper.evm.opcodes import DEFAULT_EVM_VERSION, evm_wrapper from vyper.typing import ( ContractCodes, @@ -54,7 +54,7 @@ def compile_codes( exc_handler: Union[Callable, None] = None, interface_codes: Union[InterfaceDict, InterfaceImports, None] = None, initial_id: int = 0, - optimize: OptimizerLevel = OptimizerLevel.GAS, + optimize: OptimizationLevel = OptimizationLevel.GAS, storage_layouts: Dict[ContractPath, StorageLayout] = None, show_gas_estimates: bool = False, no_bytecode_metadata: bool = False, @@ -77,7 +77,7 @@ def compile_codes( evm_version: str, optional The target EVM ruleset to compile for. If not given, defaults to the latest implemented ruleset. - optimize: OptimizerLevel, optional + optimize: OptimizationLevel, optional Set optimization mode. Defaults to OptimizationLevel.GAS show_gas_estimates: bool, optional Show gas estimates for abi and ir output modes @@ -155,7 +155,7 @@ def compile_code( output_formats: Optional[OutputFormats] = None, interface_codes: Optional[InterfaceImports] = None, evm_version: str = DEFAULT_EVM_VERSION, - optimize: OptimizerLevel = OptimizerLevel.GAS, + optimize: OptimizationLevel = OptimizationLevel.GAS, storage_layout_override: StorageLayout = None, show_gas_estimates: bool = False, ) -> dict: @@ -172,7 +172,7 @@ def compile_code( evm_version: str, optional The target EVM ruleset to compile for. If not given, defaults to the latest implemented ruleset. - optimize: OptimizerLevel, optional + optimize: OptimizationLevel, optional Set optimization mode. Defaults to OptimizationLevel.GAS show_gas_estimates: bool, optional Show gas estimates for abi and ir output modes From df8d642345e0971252b5c1ece2fbfce57e25b821 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 7 Jul 2023 01:46:53 +0000 Subject: [PATCH 19/49] fix some tests --- tests/compiler/asm/test_asm_optimizer.py | 5 +++-- tests/conftest.py | 7 ++++++- tests/examples/factory/test_factory.py | 4 ++-- tests/parser/features/test_immutable.py | 4 +++- tests/parser/functions/test_create_functions.py | 5 ++++- tests/parser/syntax/test_address_code.py | 4 ++-- tests/parser/syntax/test_codehash.py | 6 ++---- tests/parser/types/test_dynamic_array.py | 5 +++-- 8 files changed, 25 insertions(+), 15 deletions(-) diff --git a/tests/compiler/asm/test_asm_optimizer.py b/tests/compiler/asm/test_asm_optimizer.py index f4a245e168..a0516370ff 100644 --- a/tests/compiler/asm/test_asm_optimizer.py +++ b/tests/compiler/asm/test_asm_optimizer.py @@ -1,6 +1,7 @@ import pytest from vyper.compiler.phases import CompilerData +from vyper.compiler.settings import OptimizationLevel codes = [ """ @@ -72,7 +73,7 @@ def __init__(): @pytest.mark.parametrize("code", codes) def test_dead_code_eliminator(code): - c = CompilerData(code, no_optimize=True) + c = CompilerData(code, optimize=OptimizationLevel.NONE) initcode_asm = [i for i in c.assembly if not isinstance(i, list)] runtime_asm = c.assembly_runtime @@ -87,7 +88,7 @@ def test_dead_code_eliminator(code): for s in (ctor_only_label, runtime_only_label): assert s + "_runtime" in runtime_asm - c = CompilerData(code, no_optimize=False) + c = CompilerData(code, optimize=OptimizationLevel.GAS) initcode_asm = [i for i in c.assembly if not isinstance(i, list)] runtime_asm = c.assembly_runtime diff --git a/tests/conftest.py b/tests/conftest.py index 015ccae177..b083e00280 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,7 +37,12 @@ def set_evm_verbose_logging(): def pytest_addoption(parser): - parser.addoption("--optimize", choices=["codesize", "gas", "none"], default="gas", help="disable asm and IR optimizations") + parser.addoption( + "--optimize", + choices=["codesize", "gas", "none"], + default="gas", + help="disable asm and IR optimizations", + ) @pytest.fixture(scope="module") diff --git a/tests/examples/factory/test_factory.py b/tests/examples/factory/test_factory.py index 15becc05f1..49847729d7 100644 --- a/tests/examples/factory/test_factory.py +++ b/tests/examples/factory/test_factory.py @@ -30,12 +30,12 @@ def create_exchange(token, factory): @pytest.fixture -def factory(get_contract, no_optimize): +def factory(get_contract, optimize): with open("examples/factory/Exchange.vy") as f: code = f.read() exchange_interface = vyper.compile_code( - code, output_formats=["bytecode_runtime"], no_optimize=no_optimize + code, output_formats=["bytecode_runtime"], optimize=optimize ) exchange_deployed_bytecode = exchange_interface["bytecode_runtime"] diff --git a/tests/parser/features/test_immutable.py b/tests/parser/features/test_immutable.py index 7300d0f2d9..47f7fc748e 100644 --- a/tests/parser/features/test_immutable.py +++ b/tests/parser/features/test_immutable.py @@ -1,5 +1,7 @@ import pytest +from vyper.compiler.settings import OptimizationLevel + @pytest.mark.parametrize( "typ,value", @@ -269,7 +271,7 @@ def __init__(to_copy: address): # GH issue 3101, take 2 def test_immutables_initialized2(get_contract, get_contract_from_ir): dummy_contract = get_contract_from_ir( - ["deploy", 0, ["seq"] + ["invalid"] * 600, 0], no_optimize=True + ["deploy", 0, ["seq"] + ["invalid"] * 600, 0], optimize=OptimizationLevel.NONE ) # rekt because immutables section extends past allocated memory diff --git a/tests/parser/functions/test_create_functions.py b/tests/parser/functions/test_create_functions.py index 64e0823146..876d50b27d 100644 --- a/tests/parser/functions/test_create_functions.py +++ b/tests/parser/functions/test_create_functions.py @@ -5,6 +5,7 @@ import vyper.ir.compile_ir as compile_ir from vyper.codegen.ir_node import IRnode +from vyper.compiler.settings import OptimizationLevel from vyper.utils import EIP_170_LIMIT, checksum_encode, keccak256 @@ -232,7 +233,9 @@ def test(code_ofst: uint256) -> address: # zeroes (so no matter which offset, create_from_blueprint will # return empty code) ir = IRnode.from_list(["deploy", 0, ["seq"] + ["stop"] * initcode_len, 0]) - bytecode, _ = compile_ir.assembly_to_evm(compile_ir.compile_to_assembly(ir, no_optimize=True)) + bytecode, _ = compile_ir.assembly_to_evm( + compile_ir.compile_to_assembly(ir, optimize=OptimizationLevel.NONE) + ) # manually deploy the bytecode c = w3.eth.contract(abi=[], bytecode=bytecode) deploy_transaction = c.constructor() diff --git a/tests/parser/syntax/test_address_code.py b/tests/parser/syntax/test_address_code.py index 25fe1be0b4..0e442fdb36 100644 --- a/tests/parser/syntax/test_address_code.py +++ b/tests/parser/syntax/test_address_code.py @@ -161,7 +161,7 @@ def test_address_code_compile_success(code: str): compiler.compile_code(code) -def test_address_code_self_success(get_contract, no_optimize: bool): +def test_address_code_self_success(get_contract, optimize): code = """ code_deployment: public(Bytes[32]) @@ -175,7 +175,7 @@ def code_runtime() -> Bytes[32]: """ contract = get_contract(code) code_compiled = compiler.compile_code( - code, output_formats=["bytecode", "bytecode_runtime"], no_optimize=no_optimize + code, output_formats=["bytecode", "bytecode_runtime"], optimize=optimize ) assert contract.code_deployment() == bytes.fromhex(code_compiled["bytecode"][2:])[:32] assert contract.code_runtime() == bytes.fromhex(code_compiled["bytecode_runtime"][2:])[:32] diff --git a/tests/parser/syntax/test_codehash.py b/tests/parser/syntax/test_codehash.py index e4b6d90d8d..8185683514 100644 --- a/tests/parser/syntax/test_codehash.py +++ b/tests/parser/syntax/test_codehash.py @@ -6,7 +6,7 @@ @pytest.mark.parametrize("evm_version", list(EVM_VERSIONS)) -def test_get_extcodehash(get_contract, evm_version, no_optimize): +def test_get_extcodehash(get_contract, evm_version, optimize): code = """ a: address @@ -31,9 +31,7 @@ def foo3() -> bytes32: def foo4() -> bytes32: return self.a.codehash """ - compiled = compile_code( - code, ["bytecode_runtime"], evm_version=evm_version, no_optimize=no_optimize - ) + compiled = compile_code(code, ["bytecode_runtime"], evm_version=evm_version, optimize=optimize) bytecode = bytes.fromhex(compiled["bytecode_runtime"][2:]) hash_ = keccak256(bytecode) diff --git a/tests/parser/types/test_dynamic_array.py b/tests/parser/types/test_dynamic_array.py index cb55c42870..cbae183fe4 100644 --- a/tests/parser/types/test_dynamic_array.py +++ b/tests/parser/types/test_dynamic_array.py @@ -2,6 +2,7 @@ import pytest +from vyper.compiler.settings import OptimizationLevel from vyper.exceptions import ( ArgumentException, ArrayIndexException, @@ -1543,7 +1544,7 @@ def bar(x: int128) -> DynArray[int128, 3]: assert c.bar(7) == [7, 14] -def test_nested_struct_of_lists(get_contract, assert_compile_failed, no_optimize): +def test_nested_struct_of_lists(get_contract, assert_compile_failed, optimize): code = """ struct nestedFoo: a1: DynArray[DynArray[DynArray[uint256, 2], 2], 2] @@ -1585,7 +1586,7 @@ def bar2() -> uint256: newFoo.b1[0][1][0].a1[0][0][0] """ - if no_optimize: + if optimize == OptimizationLevel.NONE: # fails at assembly stage with too many stack variables assert_compile_failed(lambda: get_contract(code), Exception) else: From 0640d7e76ba35bb092bd502b1eb9034b69fe0623 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 7 Jul 2023 19:58:12 -0400 Subject: [PATCH 20/49] source code pragma for compiler modes --- vyper/ast/pre_parser.py | 54 ++++++++++++++++++++++++------ vyper/ast/utils.py | 7 ++-- vyper/cli/utils.py | 2 +- vyper/cli/vyper_compile.py | 28 +++++++++------- vyper/cli/vyper_json.py | 22 ++++++++----- vyper/compiler/__init__.py | 67 +++++++++++++++++++------------------- vyper/compiler/phases.py | 59 ++++++++++++++++++++++++++------- vyper/compiler/settings.py | 20 ++++++++---- vyper/evm/opcodes.py | 22 ++++++------- 9 files changed, 184 insertions(+), 97 deletions(-) diff --git a/vyper/ast/pre_parser.py b/vyper/ast/pre_parser.py index f29150a5d3..c57d7896a4 100644 --- a/vyper/ast/pre_parser.py +++ b/vyper/ast/pre_parser.py @@ -5,7 +5,12 @@ from semantic_version import NpmSpec, Version -from vyper.exceptions import SyntaxException, VersionException +from vyper.compiler.settings import OptimizationLevel, Settings + +# seems a bit early to be importing this but we want it to validate the +# evm-version pragma +from vyper.evm.opcodes import EVM_VERSIONS +from vyper.exceptions import StructureException, SyntaxException, VersionException from vyper.typing import ModificationOffsets, ParserPosition VERSION_ALPHA_RE = re.compile(r"(?<=\d)a(?=\d)") # 0.1.0a17 @@ -33,10 +38,7 @@ def validate_version_pragma(version_str: str, start: ParserPosition) -> None: # NOTE: should be `x.y.z.*` installed_version = ".".join(__version__.split(".")[:3]) - version_arr = version_str.split("@version") - - raw_file_version = version_arr[1].strip() - strict_file_version = _convert_version_str(raw_file_version) + strict_file_version = _convert_version_str(version_str) strict_compiler_version = Version(_convert_version_str(installed_version)) if len(strict_file_version) == 0: @@ -46,14 +48,14 @@ def validate_version_pragma(version_str: str, start: ParserPosition) -> None: npm_spec = NpmSpec(strict_file_version) except ValueError: raise VersionException( - f'Version specification "{raw_file_version}" is not a valid NPM semantic ' + f'Version specification "{version_str}" is not a valid NPM semantic ' f"version specification", start, ) if not npm_spec.match(strict_compiler_version): raise VersionException( - f'Version specification "{raw_file_version}" is not compatible ' + f'Version specification "{version_str}" is not compatible ' f'with compiler version "{installed_version}"', start, ) @@ -93,6 +95,7 @@ def pre_parse(code: str) -> Tuple[ModificationOffsets, str]: """ result = [] modification_offsets: ModificationOffsets = {} + settings = Settings() try: code_bytes = code.encode("utf-8") @@ -108,8 +111,39 @@ def pre_parse(code: str) -> Tuple[ModificationOffsets, str]: end = token.end line = token.line - if typ == COMMENT and "@version" in string: - validate_version_pragma(string[1:], start) + if typ == COMMENT: + contents = string[1:].strip() + if contents.startswith("@version "): + if settings.compiler_version is not None: + raise StructureException("compiler version specified twice!", start) + compiler_version = contents.removeprefix("@version ").strip() + validate_version_pragma(compiler_version, start) + settings.compiler_version = compiler_version + + if string.startswith("#pragma "): + pragma = string.removeprefix("#pragma").strip() + if pragma.startswith("version "): + if settings.compiler_version is not None: + raise StructureException("pragma version specified twice!", start) + compiler_version = pragma.removeprefix("version ".strip()) + validate_version_pragma(compiler_version, start) + settings.compiler_version = compiler_version + + if pragma.startswith("optimize "): + if settings.optimize is not None: + raise StructureException("pragma optimize specified twice!", start) + try: + mode = pragma.removeprefix("optimize").strip() + settings.optimize = OptimizationLevel.from_string(mode) + except ValueError: + raise StructureException(f"Invalid optimization mode `{mode}`", start) + if pragma.startswith("evm-version "): + if settings.evm_version is not None: + raise StructureException("pragma evm-version specified twice!", start) + evm_version = pragma.removeprefix("evm-version").strip() + if evm_version not in EVM_VERSIONS: + raise StructureException("Invalid evm version: `{evm_version}`", start) + settings.evm_version = evm_version if typ == NAME and string in ("class", "yield"): raise SyntaxException( @@ -130,4 +164,4 @@ def pre_parse(code: str) -> Tuple[ModificationOffsets, str]: except TokenError as e: raise SyntaxException(e.args[0], code, e.args[1][0], e.args[1][1]) from e - return modification_offsets, untokenize(result).decode("utf-8") + return settings, modification_offsets, untokenize(result).decode("utf-8") diff --git a/vyper/ast/utils.py b/vyper/ast/utils.py index fc8aad227c..0e05a4320c 100644 --- a/vyper/ast/utils.py +++ b/vyper/ast/utils.py @@ -4,6 +4,7 @@ from vyper.ast import nodes as vy_ast from vyper.ast.annotation import annotate_python_ast from vyper.ast.pre_parser import pre_parse +from vyper.compiler.settings import Settings from vyper.exceptions import CompilerPanic, ParserException, SyntaxException @@ -12,7 +13,7 @@ def parse_to_ast( source_id: int = 0, contract_name: Optional[str] = None, add_fn_node: Optional[str] = None, -) -> vy_ast.Module: +) -> tuple[Settings, vy_ast.Module]: """ Parses a Vyper source string and generates basic Vyper AST nodes. @@ -34,7 +35,7 @@ def parse_to_ast( """ if "\x00" in source_code: raise ParserException("No null bytes (\\x00) allowed in the source code.") - class_types, reformatted_code = pre_parse(source_code) + settings, class_types, reformatted_code = pre_parse(source_code) try: py_ast = python_ast.parse(reformatted_code) except SyntaxError as e: @@ -51,7 +52,7 @@ def parse_to_ast( annotate_python_ast(py_ast, source_code, class_types, source_id, contract_name) # Convert to Vyper AST. - return vy_ast.get_node(py_ast) # type: ignore + return settings, vy_ast.get_node(py_ast) def ast_to_dict(ast_struct: Union[vy_ast.VyperNode, List]) -> Union[Dict, List]: diff --git a/vyper/cli/utils.py b/vyper/cli/utils.py index 1110ecdfdd..e3f9d48164 100644 --- a/vyper/cli/utils.py +++ b/vyper/cli/utils.py @@ -27,7 +27,7 @@ def get_interface_file_path(base_paths: Sequence, import_path: str) -> Path: def extract_file_interface_imports(code: SourceCode) -> InterfaceImports: - ast_tree = vy_ast.parse_to_ast(code) + _pragmas, ast_tree = vy_ast.parse_to_ast(code) imports_dict: InterfaceImports = {} for node in ast_tree.get_children((vy_ast.Import, vy_ast.ImportFrom)): diff --git a/vyper/cli/vyper_compile.py b/vyper/cli/vyper_compile.py index 2af2ce6493..c9fca23421 100755 --- a/vyper/cli/vyper_compile.py +++ b/vyper/cli/vyper_compile.py @@ -11,7 +11,7 @@ import vyper.codegen.ir_node as ir_node from vyper.cli import vyper_json from vyper.cli.utils import extract_file_interface_imports, get_interface_file_path -from vyper.compiler.settings import VYPER_TRACEBACK_LIMIT, OptimizationLevel +from vyper.compiler.settings import VYPER_TRACEBACK_LIMIT, OptimizationLevel, Settings from vyper.evm.opcodes import DEFAULT_EVM_VERSION, EVM_VERSIONS from vyper.typing import ContractCodes, ContractPath, OutputFormats @@ -102,7 +102,6 @@ def _parse_args(argv): help=f"Select desired EVM version (default {DEFAULT_EVM_VERSION}). " "note: cancun support is EXPERIMENTAL", choices=list(EVM_VERSIONS), - default=DEFAULT_EVM_VERSION, dest="evm_version", ) parser.add_argument("--no-optimize", help="Do not optimize", action="store_true") @@ -155,18 +154,25 @@ def _parse_args(argv): if args.no_optimize and args.optimize: raise ValueError("Cannot use `--no-optimize` and `--optimize` at the same time!") - if args.no_optimize or not args.optimize: - optimize = OptimizationLevel.NONE - else: - optimize = OptimizationLevel.from_string(args.optimize) + settings = Settings() + + if args.no_optimize: + settings.optimize = OptimizationLevel.NONE + elif args.optimize is not None: + settings.optimize = OptimizationLevel.from_string(args.optimize) + + if args.evm_version: + settings.evm_version = args.evm_version + + if args.verbose: + print(f"using `{settings}`", file=sys.stderr) compiled = compile_files( args.input_files, output_formats, args.root_folder, args.show_gas_estimates, - args.evm_version, - optimize, + settings, args.storage_layout, args.no_bytecode_metadata, ) @@ -260,8 +266,7 @@ def compile_files( output_formats: OutputFormats, root_folder: str = ".", show_gas_estimates: bool = False, - evm_version: str = DEFAULT_EVM_VERSION, - optimize: OptimizationLevel = OptimizationLevel.GAS, + settings: Settings = None, storage_layout: Iterable[str] = None, no_bytecode_metadata: bool = False, ) -> OrderedDict: @@ -303,8 +308,7 @@ def compile_files( final_formats, exc_handler=exc_handler, interface_codes=get_interface_codes(root_path, contract_sources), - evm_version=evm_version, - optimize=optimize, + settings=settings, storage_layouts=storage_layouts, show_gas_estimates=show_gas_estimates, no_bytecode_metadata=no_bytecode_metadata, diff --git a/vyper/cli/vyper_json.py b/vyper/cli/vyper_json.py index bc71bc3fd0..cc5cf3674f 100755 --- a/vyper/cli/vyper_json.py +++ b/vyper/cli/vyper_json.py @@ -9,8 +9,8 @@ import vyper from vyper.cli.utils import extract_file_interface_imports, get_interface_file_path -from vyper.compiler.settings import OptimizationLevel -from vyper.evm.opcodes import DEFAULT_EVM_VERSION, EVM_VERSIONS +from vyper.compiler.settings import OptimizationLevel, Settings +from vyper.evm.opcodes import EVM_VERSIONS from vyper.exceptions import JSONError from vyper.typing import ContractCodes, ContractPath from vyper.utils import keccak256 @@ -147,9 +147,10 @@ def _standardize_path(path_str: str) -> str: def get_evm_version(input_dict: Dict) -> str: if "settings" not in input_dict: - return DEFAULT_EVM_VERSION + return None - evm_version = input_dict["settings"].get("evmVersion", DEFAULT_EVM_VERSION) + # TODO: move this validation somewhere it can be reused more easily + evm_version = input_dict["settings"].get("evmVersion") if evm_version in ( "homestead", "tangerineWhistle", @@ -360,17 +361,21 @@ def compile_from_input_dict( if input_dict["language"] != "Vyper": raise JSONError(f"Invalid language '{input_dict['language']}' - Only Vyper is supported.") - evm_version = get_evm_version(input_dict) + evm_version = input_dict.get("evm_version") - optimize = input_dict["settings"].get("optimize", "gas") + optimize = input_dict["settings"].get("optimize") if isinstance(optimize, bool): # bool optimization level for backwards compatibility warnings.warn( "optimize: is deprecated! please use one of 'gas', 'codesize', 'none'." ) optimize = OptimizationLevel.GAS if optimize else OptimizationLevel.NONE - else: + elif isinstance(optimize, str): optimize = OptimizationLevel.from_string(optimize) + else: + assert optimize is None + + settings = Settings(evm_version=evm_version, optimize=optimize) no_bytecode_metadata = not input_dict["settings"].get("bytecodeMetadata", True) @@ -394,8 +399,7 @@ def compile_from_input_dict( output_formats[contract_path], interface_codes=interface_codes, initial_id=id_, - optimize=optimize, - evm_version=evm_version, + settings=settings, no_bytecode_metadata=no_bytecode_metadata, ) except Exception as exc: diff --git a/vyper/compiler/__init__.py b/vyper/compiler/__init__.py index fae0641ecf..19fc79c23c 100644 --- a/vyper/compiler/__init__.py +++ b/vyper/compiler/__init__.py @@ -5,8 +5,8 @@ import vyper.codegen.core as codegen import vyper.compiler.output as output from vyper.compiler.phases import CompilerData -from vyper.compiler.settings import OptimizationLevel -from vyper.evm.opcodes import DEFAULT_EVM_VERSION, evm_wrapper +from vyper.compiler.settings import Settings +from vyper.evm.opcodes import DEFAULT_EVM_VERSION, anchor_evm_version from vyper.typing import ( ContractCodes, ContractPath, @@ -47,14 +47,13 @@ } -@evm_wrapper def compile_codes( contract_sources: ContractCodes, output_formats: Union[OutputDict, OutputFormats, None] = None, exc_handler: Union[Callable, None] = None, interface_codes: Union[InterfaceDict, InterfaceImports, None] = None, initial_id: int = 0, - optimize: OptimizationLevel = OptimizationLevel.GAS, + settings: Settings = None, storage_layouts: Dict[ContractPath, StorageLayout] = None, show_gas_estimates: bool = False, no_bytecode_metadata: bool = False, @@ -74,11 +73,8 @@ def compile_codes( two arguments - the name of the contract, and the exception that was raised initial_id: int, optional The lowest source ID value to be used when generating the source map. - evm_version: str, optional - The target EVM ruleset to compile for. If not given, defaults to the latest - implemented ruleset. - optimize: OptimizationLevel, optional - Set optimization mode. Defaults to OptimizationLevel.GAS + settings: Settings, optional + Compiler settings show_gas_estimates: bool, optional Show gas estimates for abi and ir output modes interface_codes: Dict, optional @@ -100,6 +96,8 @@ def compile_codes( Compiler output as `{'contract name': {'output key': "output data"}}` """ + settings = settings or None + if output_formats is None: output_formats = ("bytecode",) if isinstance(output_formats, Sequence): @@ -122,27 +120,30 @@ def compile_codes( # make IR output the same between runs codegen.reset_names() - compiler_data = CompilerData( - source_code, - contract_name, - interfaces, - source_id, - optimize, - storage_layout_override, - show_gas_estimates, - no_bytecode_metadata, - ) - for output_format in output_formats[contract_name]: - if output_format not in OUTPUT_FORMATS: - raise ValueError(f"Unsupported format type {repr(output_format)}") - try: - out.setdefault(contract_name, {}) - out[contract_name][output_format] = OUTPUT_FORMATS[output_format](compiler_data) - except Exception as exc: - if exc_handler is not None: - exc_handler(contract_name, exc) - else: - raise exc + + with anchor_evm_version(settings.evm_version): + compiler_data = CompilerData( + source_code, + contract_name, + interfaces, + source_id, + settings, + storage_layout_override, + show_gas_estimates, + no_bytecode_metadata, + ) + for output_format in output_formats[contract_name]: + if output_format not in OUTPUT_FORMATS: + raise ValueError(f"Unsupported format type {repr(output_format)}") + try: + out.setdefault(contract_name, {}) + formatter = OUTPUT_FORMATS[output_format] + out[contract_name][output_format] = formatter(compiler_data) + except Exception as exc: + if exc_handler is not None: + exc_handler(contract_name, exc) + else: + raise exc return out @@ -154,8 +155,7 @@ def compile_code( contract_source: str, output_formats: Optional[OutputFormats] = None, interface_codes: Optional[InterfaceImports] = None, - evm_version: str = DEFAULT_EVM_VERSION, - optimize: OptimizationLevel = OptimizationLevel.GAS, + settings: Settings = None, storage_layout_override: StorageLayout = None, show_gas_estimates: bool = False, ) -> dict: @@ -195,8 +195,7 @@ def compile_code( contract_sources, output_formats, interface_codes=interface_codes, - evm_version=evm_version, - optimize=optimize, + settings=settings, storage_layouts=storage_layouts, show_gas_estimates=show_gas_estimates, )[UNKNOWN_CONTRACT_NAME] diff --git a/vyper/compiler/phases.py b/vyper/compiler/phases.py index 1a9e65f88c..99eddea1a2 100644 --- a/vyper/compiler/phases.py +++ b/vyper/compiler/phases.py @@ -7,7 +7,7 @@ from vyper.codegen import module from vyper.codegen.global_context import GlobalContext from vyper.codegen.ir_node import IRnode -from vyper.compiler.settings import OptimizationLevel +from vyper.compiler.settings import OptimizationLevel, Settings from vyper.ir import compile_ir, optimizer from vyper.semantics import set_data_positions, validate_semantics from vyper.semantics.types.function import ContractFunctionT @@ -50,7 +50,7 @@ def __init__( contract_name: str = "VyperContract", interface_codes: Optional[InterfaceImports] = None, source_id: int = 0, - optimize: OptimizationLevel = OptimizationLevel.GAS, + settings: Settings = None, storage_layout: StorageLayout = None, show_gas_estimates: bool = False, no_bytecode_metadata: bool = False, @@ -70,8 +70,8 @@ def __init__( * JSON interfaces are given as lists, vyper interfaces as strings source_id : int, optional ID number used to identify this contract in the source map. - optimize: OptimizationLevel, optional - Set optimization mode. Defaults to OptimizationLevel.GAS + settings: Settings + Set optimization mode. show_gas_estimates: bool, optional Show gas estimates for abi and ir output modes no_bytecode_metadata: bool, optional @@ -81,14 +81,49 @@ def __init__( self.source_code = source_code self.interface_codes = interface_codes self.source_id = source_id - self.optimize = optimize self.storage_layout_override = storage_layout self.show_gas_estimates = show_gas_estimates self.no_bytecode_metadata = no_bytecode_metadata + self.settings = settings or Settings() @cached_property - def vyper_module(self) -> vy_ast.Module: - return generate_ast(self.source_code, self.source_id, self.contract_name) + def _generate_ast(self): + settings, ast = generate_ast(self.source_code, self.source_id, self.contract_name) + # validate the compiler settings + # XXX: this is a bit ugly, clean up later + if settings.evm_version is not None: + if ( + self.settings.evm_version is not None + and self.settings.evm_version != settings.evm_version + ): + # TODO: consider raising an exception + warnings.warn( + f"compiler settings indicate evm version {self.settings.evm_version}, " + f"but source pragma indicates {settings.evm_version}.\n" + f"using `evm version: {self.settings.evm_version}`!" + ) + + self.settings.evm_version = settings.evm_version + + if settings.optimize is not None: + if self.settings.optimize is not None and self.settings.optimize != settings.optimize: + # TODO: consider raising an exception + warnings.warn( + f"compiler options indicate optimization mode {self.settings.optimize}, " + f"but source pragma indicates {settings.optimize}.\n" + f"using `optimize: {self.settings.optimize}`!" + ) + self.settings.optimize = settings.optimize + + # ensure defaults + if self.settings.optimize is None: + self.settings.optimize = OptimizationLevel.default() + + return ast + + @cached_property + def vyper_module(self): + return self._generate_ast @cached_property def vyper_module_unfolded(self) -> vy_ast.Module: @@ -120,7 +155,7 @@ def global_ctx(self) -> GlobalContext: @cached_property def _ir_output(self): # fetch both deployment and runtime IR - return generate_ir_nodes(self.global_ctx, self.optimize) + return generate_ir_nodes(self.global_ctx, self.settings.optimize) @property def ir_nodes(self) -> IRnode: @@ -143,11 +178,11 @@ def function_signatures(self) -> dict[str, ContractFunctionT]: @cached_property def assembly(self) -> list: - return generate_assembly(self.ir_nodes, self.optimize) + return generate_assembly(self.ir_nodes, self.settings.optimize) @cached_property def assembly_runtime(self) -> list: - return generate_assembly(self.ir_runtime, self.optimize) + return generate_assembly(self.ir_runtime, self.settings.optimize) @cached_property def bytecode(self) -> bytes: @@ -170,7 +205,9 @@ def blueprint_bytecode(self) -> bytes: return deploy_bytecode + blueprint_bytecode -def generate_ast(source_code: str, source_id: int, contract_name: str) -> vy_ast.Module: +def generate_ast( + source_code: str, source_id: int, contract_name: str +) -> tuple[Settings, vy_ast.Module]: """ Generate a Vyper AST from source code. diff --git a/vyper/compiler/settings.py b/vyper/compiler/settings.py index 1647bb5691..bb5e9cdc25 100644 --- a/vyper/compiler/settings.py +++ b/vyper/compiler/settings.py @@ -1,4 +1,5 @@ import os +from dataclasses import dataclass from enum import Enum from typing import Optional @@ -16,9 +17,9 @@ class OptimizationLevel(Enum): - NONE = 0 - GAS = 1 - CODESIZE = 2 + NONE = 1 + GAS = 2 + CODESIZE = 3 @classmethod def from_string(cls, val): @@ -31,6 +32,13 @@ def from_string(cls, val): return cls.CODESIZE raise ValueError(f"unrecognized optimization level: {val}") - # @classmethod - # def default(cls): - # return cls.GAS + @classmethod + def default(cls): + return cls.GAS + + +@dataclass +class Settings: + compiler_version: Optional[str] = None + optimize: Optional[OptimizationLevel] = None + evm_version: Optional[str] = None diff --git a/vyper/evm/opcodes.py b/vyper/evm/opcodes.py index 7550d047b5..88a29df1cf 100644 --- a/vyper/evm/opcodes.py +++ b/vyper/evm/opcodes.py @@ -1,3 +1,4 @@ +import contextlib from typing import Dict, Optional from vyper.exceptions import CompilerPanic @@ -206,17 +207,16 @@ IR_OPCODES: OpcodeMap = {**OPCODES, **PSEUDO_OPCODES} -def evm_wrapper(fn, *args, **kwargs): - def _wrapper(*args, **kwargs): - global active_evm_version - evm_version = kwargs.pop("evm_version", None) or DEFAULT_EVM_VERSION - active_evm_version = EVM_VERSIONS[evm_version] - try: - return fn(*args, **kwargs) - finally: - active_evm_version = EVM_VERSIONS[DEFAULT_EVM_VERSION] - - return _wrapper +@contextlib.contextmanager +def anchor_evm_version(evm_version: str = None): + global active_evm_version + anchor = active_evm_version + evm_version = evm_version or DEFAULT_EVM_VERSION + active_evm_version = EVM_VERSIONS[evm_version] + try: + yield + finally: + active_evm_version = anchor def _gas(value: OpcodeValue, idx: int) -> Optional[OpcodeRulesetValue]: From 2c7d696af9fb590e822ef09711358141991cbd6e Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 10 Jul 2023 13:39:02 -0400 Subject: [PATCH 21/49] fix mypy and some lint note: mypy needed bump to 0.940 to handle match/case --- setup.py | 2 +- vyper/ast/pre_parser.py | 3 +-- vyper/ast/utils.py | 4 +++- vyper/cli/vyper_compile.py | 6 +++--- vyper/cli/vyper_json.py | 6 +++--- vyper/codegen/global_context.py | 3 +-- vyper/compiler/__init__.py | 7 +++---- vyper/compiler/phases.py | 5 ++--- vyper/evm/opcodes.py | 4 ++-- vyper/semantics/analysis/module.py | 2 +- 10 files changed, 20 insertions(+), 22 deletions(-) diff --git a/setup.py b/setup.py index 05cb52259d..4cacaeca18 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ "flake8-bugbear==20.1.4", "flake8-use-fstring==1.1", "isort==5.9.3", - "mypy==0.910", + "mypy==0.940", ], "docs": ["recommonmark", "sphinx>=6.0,<7.0", "sphinx_rtd_theme>=1.2,<1.3"], "dev": ["ipython", "pre-commit", "pyinstaller", "twine"], diff --git a/vyper/ast/pre_parser.py b/vyper/ast/pre_parser.py index c57d7896a4..9a6af0d981 100644 --- a/vyper/ast/pre_parser.py +++ b/vyper/ast/pre_parser.py @@ -1,7 +1,6 @@ import io import re from tokenize import COMMENT, NAME, OP, TokenError, TokenInfo, tokenize, untokenize -from typing import Tuple from semantic_version import NpmSpec, Version @@ -68,7 +67,7 @@ def validate_version_pragma(version_str: str, start: ParserPosition) -> None: VYPER_EXPRESSION_TYPES = {"log"} -def pre_parse(code: str) -> Tuple[ModificationOffsets, str]: +def pre_parse(code: str) -> tuple[Settings, ModificationOffsets, str]: """ Re-formats a vyper source string into a python source string and performs some validation. More specifically, diff --git a/vyper/ast/utils.py b/vyper/ast/utils.py index 0e05a4320c..ea0e87e509 100644 --- a/vyper/ast/utils.py +++ b/vyper/ast/utils.py @@ -52,7 +52,9 @@ def parse_to_ast( annotate_python_ast(py_ast, source_code, class_types, source_id, contract_name) # Convert to Vyper AST. - return settings, vy_ast.get_node(py_ast) + module = vy_ast.get_node(py_ast) + assert isinstance(module, vy_ast.Module) # mypy hint + return settings, module def ast_to_dict(ast_struct: Union[vy_ast.VyperNode, List]) -> Union[Dict, List]: diff --git a/vyper/cli/vyper_compile.py b/vyper/cli/vyper_compile.py index c9fca23421..71e78dd666 100755 --- a/vyper/cli/vyper_compile.py +++ b/vyper/cli/vyper_compile.py @@ -5,7 +5,7 @@ import warnings from collections import OrderedDict from pathlib import Path -from typing import Dict, Iterable, Iterator, Set, TypeVar +from typing import Dict, Iterable, Iterator, Optional, Set, TypeVar import vyper import vyper.codegen.ir_node as ir_node @@ -266,8 +266,8 @@ def compile_files( output_formats: OutputFormats, root_folder: str = ".", show_gas_estimates: bool = False, - settings: Settings = None, - storage_layout: Iterable[str] = None, + settings: Optional[Settings] = None, + storage_layout: Optional[Iterable[str]] = None, no_bytecode_metadata: bool = False, ) -> OrderedDict: root_path = Path(root_folder).resolve() diff --git a/vyper/cli/vyper_json.py b/vyper/cli/vyper_json.py index cc5cf3674f..dbb590cad8 100755 --- a/vyper/cli/vyper_json.py +++ b/vyper/cli/vyper_json.py @@ -5,7 +5,7 @@ import sys import warnings from pathlib import Path -from typing import Any, Callable, Dict, Hashable, List, Tuple, Union +from typing import Any, Callable, Dict, Hashable, List, Optional, Tuple, Union import vyper from vyper.cli.utils import extract_file_interface_imports, get_interface_file_path @@ -145,7 +145,7 @@ def _standardize_path(path_str: str) -> str: return path.as_posix() -def get_evm_version(input_dict: Dict) -> str: +def get_evm_version(input_dict: Dict) -> Optional[str]: if "settings" not in input_dict: return None @@ -361,7 +361,7 @@ def compile_from_input_dict( if input_dict["language"] != "Vyper": raise JSONError(f"Invalid language '{input_dict['language']}' - Only Vyper is supported.") - evm_version = input_dict.get("evm_version") + evm_version = get_evm_version(input_dict) optimize = input_dict["settings"].get("optimize") if isinstance(optimize, bool): diff --git a/vyper/codegen/global_context.py b/vyper/codegen/global_context.py index 613c1a1d2f..1f6783f6f8 100644 --- a/vyper/codegen/global_context.py +++ b/vyper/codegen/global_context.py @@ -2,13 +2,12 @@ from typing import Optional from vyper import ast as vy_ast -from vyper.compiler.settings import OptimizationLevel # Datatype to store all global context information. # TODO: rename me to ModuleT class GlobalContext: - def __init__(self, module: Optional[vy_ast.Module] = None, optimize=OptimizationLevel.GAS): + def __init__(self, module: Optional[vy_ast.Module] = None): self._module = module @cached_property diff --git a/vyper/compiler/__init__.py b/vyper/compiler/__init__.py index 19fc79c23c..ddcc560d1d 100644 --- a/vyper/compiler/__init__.py +++ b/vyper/compiler/__init__.py @@ -54,7 +54,7 @@ def compile_codes( interface_codes: Union[InterfaceDict, InterfaceImports, None] = None, initial_id: int = 0, settings: Settings = None, - storage_layouts: Dict[ContractPath, StorageLayout] = None, + storage_layouts: Optional[dict[ContractPath, Optional[StorageLayout]]] = None, show_gas_estimates: bool = False, no_bytecode_metadata: bool = False, ) -> OrderedDict: @@ -95,8 +95,7 @@ def compile_codes( Dict Compiler output as `{'contract name': {'output key': "output data"}}` """ - - settings = settings or None + settings = settings or Settings() if output_formats is None: output_formats = ("bytecode",) @@ -156,7 +155,7 @@ def compile_code( output_formats: Optional[OutputFormats] = None, interface_codes: Optional[InterfaceImports] = None, settings: Settings = None, - storage_layout_override: StorageLayout = None, + storage_layout_override: Optional[StorageLayout] = None, show_gas_estimates: bool = False, ) -> dict: """ diff --git a/vyper/compiler/phases.py b/vyper/compiler/phases.py index 99eddea1a2..c9c85fc34c 100644 --- a/vyper/compiler/phases.py +++ b/vyper/compiler/phases.py @@ -298,9 +298,7 @@ def generate_ir_nodes(global_ctx: GlobalContext, optimize: bool) -> tuple[IRnode return ir_nodes, ir_runtime -def generate_assembly( - ir_nodes: IRnode, optimize: OptimizationLevel = OptimizationLevel.GAS -) -> list: +def generate_assembly(ir_nodes: IRnode, optimize: Optional[OptimizationLevel] = None) -> list: """ Generate assembly instructions from IR. @@ -314,6 +312,7 @@ def generate_assembly( list List of assembly instructions. """ + optimize = optimize or OptimizationLevel.GAS assembly = compile_ir.compile_to_assembly(ir_nodes, optimize=optimize) if _find_nested_opcode(assembly, "DEBUG"): diff --git a/vyper/evm/opcodes.py b/vyper/evm/opcodes.py index 88a29df1cf..4fec13e897 100644 --- a/vyper/evm/opcodes.py +++ b/vyper/evm/opcodes.py @@ -1,5 +1,5 @@ import contextlib -from typing import Dict, Optional +from typing import Dict, Generator, Optional from vyper.exceptions import CompilerPanic from vyper.typing import OpcodeGasCost, OpcodeMap, OpcodeRulesetMap, OpcodeRulesetValue, OpcodeValue @@ -208,7 +208,7 @@ @contextlib.contextmanager -def anchor_evm_version(evm_version: str = None): +def anchor_evm_version(evm_version: Optional[str] = None) -> Generator: global active_evm_version anchor = active_evm_version evm_version = evm_version or DEFAULT_EVM_VERSION diff --git a/vyper/semantics/analysis/module.py b/vyper/semantics/analysis/module.py index d916dcf119..15db605587 100644 --- a/vyper/semantics/analysis/module.py +++ b/vyper/semantics/analysis/module.py @@ -337,7 +337,7 @@ def _add_import( raise UndeclaredDefinition(f"Unknown interface: {name}. {suggestions_str}", node) if interface_codes[name]["type"] == "vyper": - interface_ast = vy_ast.parse_to_ast(interface_codes[name]["code"], contract_name=name) + _, interface_ast = vy_ast.parse_to_ast(interface_codes[name]["code"], contract_name=name) type_ = InterfaceT.from_ast(interface_ast) elif interface_codes[name]["type"] == "json": type_ = InterfaceT.from_json_abi(name, interface_codes[name]["code"]) # type: ignore From be5f36ac03d11d8c139b4a7fac071119416eaae4 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 10 Jul 2023 18:03:52 +0000 Subject: [PATCH 22/49] fix tests --- tests/ast/test_pre_parser.py | 8 ++++---- tests/base_conftest.py | 13 +++++++++---- tests/cli/vyper_json/test_get_settings.py | 5 ----- tests/compiler/asm/test_asm_optimizer.py | 6 +++--- tests/examples/factory/test_factory.py | 3 ++- tests/parser/features/test_transient.py | 15 ++++++++------- .../test_annotate_and_optimize_ast.py | 2 +- tests/parser/syntax/test_address_code.py | 4 +++- tests/parser/syntax/test_chainid.py | 4 +++- tests/parser/syntax/test_codehash.py | 4 +++- tests/parser/syntax/test_self_balance.py | 4 +++- vyper/ast/__init__.py | 2 +- vyper/ast/nodes.pyi | 2 +- vyper/ast/pre_parser.py | 2 +- vyper/ast/utils.py | 8 ++++++-- vyper/cli/utils.py | 2 +- vyper/cli/vyper_json.py | 3 +++ vyper/compiler/phases.py | 2 +- vyper/semantics/analysis/module.py | 2 +- 19 files changed, 54 insertions(+), 37 deletions(-) diff --git a/tests/ast/test_pre_parser.py b/tests/ast/test_pre_parser.py index 8501bb8749..b7fd823686 100644 --- a/tests/ast/test_pre_parser.py +++ b/tests/ast/test_pre_parser.py @@ -51,14 +51,14 @@ def set_version(version): @pytest.mark.parametrize("file_version", valid_versions) def test_valid_version_pragma(file_version, mock_version): mock_version(COMPILER_VERSION) - validate_version_pragma(f" @version {file_version}", (SRC_LINE)) + validate_version_pragma(f"{file_version}", (SRC_LINE)) @pytest.mark.parametrize("file_version", invalid_versions) def test_invalid_version_pragma(file_version, mock_version): mock_version(COMPILER_VERSION) with pytest.raises(VersionException): - validate_version_pragma(f" @version {file_version}", (SRC_LINE)) + validate_version_pragma(f"{file_version}", (SRC_LINE)) prerelease_valid_versions = [ @@ -98,11 +98,11 @@ def test_invalid_version_pragma(file_version, mock_version): @pytest.mark.parametrize("file_version", prerelease_valid_versions) def test_prerelease_valid_version_pragma(file_version, mock_version): mock_version(PRERELEASE_COMPILER_VERSION) - validate_version_pragma(f" @version {file_version}", (SRC_LINE)) + validate_version_pragma(file_version, (SRC_LINE)) @pytest.mark.parametrize("file_version", prerelease_invalid_versions) def test_prerelease_invalid_version_pragma(file_version, mock_version): mock_version(PRERELEASE_COMPILER_VERSION) with pytest.raises(VersionException): - validate_version_pragma(f" @version {file_version}", (SRC_LINE)) + validate_version_pragma(file_version, (SRC_LINE)) diff --git a/tests/base_conftest.py b/tests/base_conftest.py index e509957e46..a78562e982 100644 --- a/tests/base_conftest.py +++ b/tests/base_conftest.py @@ -12,6 +12,7 @@ from vyper import compiler from vyper.ast.grammar import parse_vyper_source +from vyper.compiler.settings import Settings class VyperMethod: @@ -112,13 +113,15 @@ def w3(tester): def _get_contract(w3, source_code, optimize, *args, **kwargs): + settings = Settings() + settings.evm_version = kwargs.pop("evm_version", None) + settings.optimize = optimize out = compiler.compile_code( source_code, # test that metadata gets generated ["abi", "bytecode", "metadata"], + settings=settings, interface_codes=kwargs.pop("interface_codes", None), - optimize=optimize, - evm_version=kwargs.pop("evm_version", None), show_gas_estimates=True, # Enable gas estimates for testing ) parse_vyper_source(source_code) # Test grammar. @@ -136,12 +139,14 @@ def _get_contract(w3, source_code, optimize, *args, **kwargs): def _deploy_blueprint_for(w3, source_code, optimize, initcode_prefix=b"", **kwargs): + settings = Settings() + settings.evm_version = kwargs.pop("evm_version", None) + settings.optimize = optimize out = compiler.compile_code( source_code, ["abi", "bytecode"], interface_codes=kwargs.pop("interface_codes", None), - optimize=optimize, - evm_version=kwargs.pop("evm_version", None), + settings=settings, show_gas_estimates=True, # Enable gas estimates for testing ) parse_vyper_source(source_code) # Test grammar. diff --git a/tests/cli/vyper_json/test_get_settings.py b/tests/cli/vyper_json/test_get_settings.py index 7530e85ef8..bbe5dab113 100644 --- a/tests/cli/vyper_json/test_get_settings.py +++ b/tests/cli/vyper_json/test_get_settings.py @@ -3,7 +3,6 @@ import pytest from vyper.cli.vyper_json import get_evm_version -from vyper.evm.opcodes import DEFAULT_EVM_VERSION from vyper.exceptions import JSONError @@ -31,7 +30,3 @@ def test_early_evm(evm_version): @pytest.mark.parametrize("evm_version", ["istanbul", "berlin", "paris", "shanghai", "cancun"]) def test_valid_evm(evm_version): assert evm_version == get_evm_version({"settings": {"evmVersion": evm_version}}) - - -def test_default_evm(): - assert get_evm_version({}) == DEFAULT_EVM_VERSION diff --git a/tests/compiler/asm/test_asm_optimizer.py b/tests/compiler/asm/test_asm_optimizer.py index a0516370ff..47b70a8c70 100644 --- a/tests/compiler/asm/test_asm_optimizer.py +++ b/tests/compiler/asm/test_asm_optimizer.py @@ -1,7 +1,7 @@ import pytest from vyper.compiler.phases import CompilerData -from vyper.compiler.settings import OptimizationLevel +from vyper.compiler.settings import OptimizationLevel, Settings codes = [ """ @@ -73,7 +73,7 @@ def __init__(): @pytest.mark.parametrize("code", codes) def test_dead_code_eliminator(code): - c = CompilerData(code, optimize=OptimizationLevel.NONE) + c = CompilerData(code, settings=Settings(optimize=OptimizationLevel.NONE)) initcode_asm = [i for i in c.assembly if not isinstance(i, list)] runtime_asm = c.assembly_runtime @@ -88,7 +88,7 @@ def test_dead_code_eliminator(code): for s in (ctor_only_label, runtime_only_label): assert s + "_runtime" in runtime_asm - c = CompilerData(code, optimize=OptimizationLevel.GAS) + c = CompilerData(code, settings=Settings(optimize=OptimizationLevel.GAS)) initcode_asm = [i for i in c.assembly if not isinstance(i, list)] runtime_asm = c.assembly_runtime diff --git a/tests/examples/factory/test_factory.py b/tests/examples/factory/test_factory.py index 49847729d7..0c5cf61b04 100644 --- a/tests/examples/factory/test_factory.py +++ b/tests/examples/factory/test_factory.py @@ -2,6 +2,7 @@ from eth_utils import keccak import vyper +from vyper.compiler.settings import Settings @pytest.fixture @@ -35,7 +36,7 @@ def factory(get_contract, optimize): code = f.read() exchange_interface = vyper.compile_code( - code, output_formats=["bytecode_runtime"], optimize=optimize + code, output_formats=["bytecode_runtime"], settings=Settings(optimize=optimize) ) exchange_deployed_bytecode = exchange_interface["bytecode_runtime"] diff --git a/tests/parser/features/test_transient.py b/tests/parser/features/test_transient.py index 53354beca8..718f5ae314 100644 --- a/tests/parser/features/test_transient.py +++ b/tests/parser/features/test_transient.py @@ -1,6 +1,7 @@ import pytest from vyper.compiler import compile_code +from vyper.compiler.settings import Settings from vyper.evm.opcodes import EVM_VERSIONS from vyper.exceptions import StructureException @@ -13,20 +14,22 @@ def test_transient_blocked(evm_version): code = """ my_map: transient(HashMap[address, uint256]) """ + settings = Settings(evm_version=evm_version) if EVM_VERSIONS[evm_version] >= EVM_VERSIONS["cancun"]: - assert compile_code(code, evm_version=evm_version) is not None + assert compile_code(code, settings=settings) is not None else: with pytest.raises(StructureException): - compile_code(code, evm_version=evm_version) + compile_code(code, settings=settings) @pytest.mark.parametrize("evm_version", list(post_cancun.keys())) def test_transient_compiles(evm_version): # test transient keyword at least generates TLOAD/TSTORE opcodes + settings = Settings(evm_version=evm_version) getter_code = """ my_map: public(transient(HashMap[address, uint256])) """ - t = compile_code(getter_code, evm_version=evm_version, output_formats=["opcodes_runtime"]) + t = compile_code(getter_code, settings=settings, output_formats=["opcodes_runtime"]) t = t["opcodes_runtime"].split(" ") assert "TLOAD" in t @@ -39,7 +42,7 @@ def test_transient_compiles(evm_version): def setter(k: address, v: uint256): self.my_map[k] = v """ - t = compile_code(setter_code, evm_version=evm_version, output_formats=["opcodes_runtime"]) + t = compile_code(setter_code, settings=settings, output_formats=["opcodes_runtime"]) t = t["opcodes_runtime"].split(" ") assert "TLOAD" not in t @@ -52,9 +55,7 @@ def setter(k: address, v: uint256): def setter(k: address, v: uint256): self.my_map[k] = v """ - t = compile_code( - getter_setter_code, evm_version=evm_version, output_formats=["opcodes_runtime"] - ) + t = compile_code(getter_setter_code, settings=settings, output_formats=["opcodes_runtime"]) t = t["opcodes_runtime"].split(" ") assert "TLOAD" in t diff --git a/tests/parser/parser_utils/test_annotate_and_optimize_ast.py b/tests/parser/parser_utils/test_annotate_and_optimize_ast.py index 6f2246c6c0..68a07178bb 100644 --- a/tests/parser/parser_utils/test_annotate_and_optimize_ast.py +++ b/tests/parser/parser_utils/test_annotate_and_optimize_ast.py @@ -29,7 +29,7 @@ def foo() -> int128: def get_contract_info(source_code): - class_types, reformatted_code = pre_parse(source_code) + _, class_types, reformatted_code = pre_parse(source_code) py_ast = python_ast.parse(reformatted_code) annotate_python_ast(py_ast, reformatted_code, class_types) diff --git a/tests/parser/syntax/test_address_code.py b/tests/parser/syntax/test_address_code.py index 0e442fdb36..ff286bbdae 100644 --- a/tests/parser/syntax/test_address_code.py +++ b/tests/parser/syntax/test_address_code.py @@ -5,6 +5,7 @@ from web3 import Web3 from vyper import compiler +from vyper.compiler.settings import Settings from vyper.exceptions import NamespaceCollision, StructureException, VyperException # For reproducibility, use precompiled data of `hello: public(uint256)` using vyper 0.3.1 @@ -174,8 +175,9 @@ def code_runtime() -> Bytes[32]: return slice(self.code, 0, 32) """ contract = get_contract(code) + settings=Settings(optimize=optimize) code_compiled = compiler.compile_code( - code, output_formats=["bytecode", "bytecode_runtime"], optimize=optimize + code, output_formats=["bytecode", "bytecode_runtime"], settings=settings ) assert contract.code_deployment() == bytes.fromhex(code_compiled["bytecode"][2:])[:32] assert contract.code_runtime() == bytes.fromhex(code_compiled["bytecode_runtime"][2:])[:32] diff --git a/tests/parser/syntax/test_chainid.py b/tests/parser/syntax/test_chainid.py index be960f2fc5..2b6e08cbc4 100644 --- a/tests/parser/syntax/test_chainid.py +++ b/tests/parser/syntax/test_chainid.py @@ -1,6 +1,7 @@ import pytest from vyper import compiler +from vyper.compiler.settings import Settings from vyper.evm.opcodes import EVM_VERSIONS from vyper.exceptions import InvalidType, TypeMismatch @@ -12,8 +13,9 @@ def test_evm_version(evm_version): def foo(): a: uint256 = chain.id """ + settings = Settings(evm_version=evm_version) - assert compiler.compile_code(code, evm_version=evm_version) is not None + assert compiler.compile_code(code, settings=settings) is not None fail_list = [ diff --git a/tests/parser/syntax/test_codehash.py b/tests/parser/syntax/test_codehash.py index 8185683514..6914344e27 100644 --- a/tests/parser/syntax/test_codehash.py +++ b/tests/parser/syntax/test_codehash.py @@ -1,6 +1,7 @@ import pytest from vyper.compiler import compile_code +from vyper.compiler.settings import Settings from vyper.evm.opcodes import EVM_VERSIONS from vyper.utils import keccak256 @@ -31,7 +32,8 @@ def foo3() -> bytes32: def foo4() -> bytes32: return self.a.codehash """ - compiled = compile_code(code, ["bytecode_runtime"], evm_version=evm_version, optimize=optimize) + settings = Settings(evm_version=evm_version, optimize=optimize) + compiled = compile_code(code, ["bytecode_runtime"], settings) bytecode = bytes.fromhex(compiled["bytecode_runtime"][2:]) hash_ = keccak256(bytecode) diff --git a/tests/parser/syntax/test_self_balance.py b/tests/parser/syntax/test_self_balance.py index 949cdde324..63db58e347 100644 --- a/tests/parser/syntax/test_self_balance.py +++ b/tests/parser/syntax/test_self_balance.py @@ -1,6 +1,7 @@ import pytest from vyper import compiler +from vyper.compiler.settings import Settings from vyper.evm.opcodes import EVM_VERSIONS @@ -18,7 +19,8 @@ def get_balance() -> uint256: def __default__(): pass """ - opcodes = compiler.compile_code(code, ["opcodes"], evm_version=evm_version)["opcodes"] + settings = Settings(evm_version=evm_version) + opcodes = compiler.compile_code(code, ["opcodes"], settings=settings)["opcodes"] if EVM_VERSIONS[evm_version] >= EVM_VERSIONS["istanbul"]: assert "SELFBALANCE" in opcodes else: diff --git a/vyper/ast/__init__.py b/vyper/ast/__init__.py index 5695ceab7c..e5b81f1e7f 100644 --- a/vyper/ast/__init__.py +++ b/vyper/ast/__init__.py @@ -6,7 +6,7 @@ from . import nodes, validation from .natspec import parse_natspec from .nodes import compare_nodes -from .utils import ast_to_dict, parse_to_ast +from .utils import ast_to_dict, parse_to_ast, parse_to_ast_with_settings # adds vyper.ast.nodes classes into the local namespace for name, obj in ( diff --git a/vyper/ast/nodes.pyi b/vyper/ast/nodes.pyi index 3d83ae7506..aefb1aed51 100644 --- a/vyper/ast/nodes.pyi +++ b/vyper/ast/nodes.pyi @@ -3,7 +3,7 @@ from typing import Any, Optional, Sequence, Type, Union from .natspec import parse_natspec as parse_natspec from .utils import ast_to_dict as ast_to_dict -from .utils import parse_to_ast as parse_to_ast +from .utils import parse_to_ast as parse_to_ast, parse_to_ast_with_settings as parse_to_ast_with_settings NODE_BASE_ATTRIBUTES: Any NODE_SRC_ATTRIBUTES: Any diff --git a/vyper/ast/pre_parser.py b/vyper/ast/pre_parser.py index 9a6af0d981..35153af9d5 100644 --- a/vyper/ast/pre_parser.py +++ b/vyper/ast/pre_parser.py @@ -112,7 +112,7 @@ def pre_parse(code: str) -> tuple[Settings, ModificationOffsets, str]: if typ == COMMENT: contents = string[1:].strip() - if contents.startswith("@version "): + if contents.startswith("@version"): if settings.compiler_version is not None: raise StructureException("compiler version specified twice!", start) compiler_version = contents.removeprefix("@version ").strip() diff --git a/vyper/ast/utils.py b/vyper/ast/utils.py index ea0e87e509..949946802f 100644 --- a/vyper/ast/utils.py +++ b/vyper/ast/utils.py @@ -1,5 +1,5 @@ import ast as python_ast -from typing import Dict, List, Optional, Union +from typing import Dict, List, Optional, Union, Any from vyper.ast import nodes as vy_ast from vyper.ast.annotation import annotate_python_ast @@ -8,7 +8,11 @@ from vyper.exceptions import CompilerPanic, ParserException, SyntaxException -def parse_to_ast( +def parse_to_ast(*args: Any, **kwargs: Any) -> vy_ast.Module: + return parse_to_ast_with_settings(*args, **kwargs)[1] + + +def parse_to_ast_with_settings( source_code: str, source_id: int = 0, contract_name: Optional[str] = None, diff --git a/vyper/cli/utils.py b/vyper/cli/utils.py index e3f9d48164..1110ecdfdd 100644 --- a/vyper/cli/utils.py +++ b/vyper/cli/utils.py @@ -27,7 +27,7 @@ def get_interface_file_path(base_paths: Sequence, import_path: str) -> Path: def extract_file_interface_imports(code: SourceCode) -> InterfaceImports: - _pragmas, ast_tree = vy_ast.parse_to_ast(code) + ast_tree = vy_ast.parse_to_ast(code) imports_dict: InterfaceImports = {} for node in ast_tree.get_children((vy_ast.Import, vy_ast.ImportFrom)): diff --git a/vyper/cli/vyper_json.py b/vyper/cli/vyper_json.py index dbb590cad8..07883fec8e 100755 --- a/vyper/cli/vyper_json.py +++ b/vyper/cli/vyper_json.py @@ -151,6 +151,9 @@ def get_evm_version(input_dict: Dict) -> Optional[str]: # TODO: move this validation somewhere it can be reused more easily evm_version = input_dict["settings"].get("evmVersion") + if evm_version is None: + return None + if evm_version in ( "homestead", "tangerineWhistle", diff --git a/vyper/compiler/phases.py b/vyper/compiler/phases.py index c9c85fc34c..7859eae0d4 100644 --- a/vyper/compiler/phases.py +++ b/vyper/compiler/phases.py @@ -225,7 +225,7 @@ def generate_ast( vy_ast.Module Top-level Vyper AST node """ - return vy_ast.parse_to_ast(source_code, source_id, contract_name) + return vy_ast.parse_to_ast_with_settings(source_code, source_id, contract_name) def generate_unfolded_ast( diff --git a/vyper/semantics/analysis/module.py b/vyper/semantics/analysis/module.py index 15db605587..d916dcf119 100644 --- a/vyper/semantics/analysis/module.py +++ b/vyper/semantics/analysis/module.py @@ -337,7 +337,7 @@ def _add_import( raise UndeclaredDefinition(f"Unknown interface: {name}. {suggestions_str}", node) if interface_codes[name]["type"] == "vyper": - _, interface_ast = vy_ast.parse_to_ast(interface_codes[name]["code"], contract_name=name) + interface_ast = vy_ast.parse_to_ast(interface_codes[name]["code"], contract_name=name) type_ = InterfaceT.from_ast(interface_ast) elif interface_codes[name]["type"] == "json": type_ = InterfaceT.from_json_abi(name, interface_codes[name]["code"]) # type: ignore From d2be8f55d8456403be88aa74dac85069d44579a4 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 10 Jul 2023 18:42:54 +0000 Subject: [PATCH 23/49] remove evm_version from bitwise op tests it was probably important when we supported pre-constantinople targets, not anymore --- tests/parser/functions/test_bitwise.py | 21 ++++++++------------- tests/parser/syntax/test_codehash.py | 2 +- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/tests/parser/functions/test_bitwise.py b/tests/parser/functions/test_bitwise.py index 3e18bd292c..3ba74034ac 100644 --- a/tests/parser/functions/test_bitwise.py +++ b/tests/parser/functions/test_bitwise.py @@ -1,7 +1,6 @@ import pytest from vyper.compiler import compile_code -from vyper.evm.opcodes import EVM_VERSIONS from vyper.exceptions import InvalidLiteral, InvalidOperation, TypeMismatch from vyper.utils import unsigned_to_signed @@ -32,16 +31,14 @@ def _shr(x: uint256, y: uint256) -> uint256: """ -@pytest.mark.parametrize("evm_version", list(EVM_VERSIONS)) -def test_bitwise_opcodes(evm_version): - opcodes = compile_code(code, ["opcodes"], evm_version=evm_version)["opcodes"] +def test_bitwise_opcodes(): + opcodes = compile_code(code, ["opcodes"])["opcodes"] assert "SHL" in opcodes assert "SHR" in opcodes -@pytest.mark.parametrize("evm_version", list(EVM_VERSIONS)) -def test_test_bitwise(get_contract_with_gas_estimation, evm_version): - c = get_contract_with_gas_estimation(code, evm_version=evm_version) +def test_test_bitwise(get_contract_with_gas_estimation): + c = get_contract_with_gas_estimation(code) x = 126416208461208640982146408124 y = 7128468721412412459 assert c._bitwise_and(x, y) == (x & y) @@ -55,8 +52,7 @@ def test_test_bitwise(get_contract_with_gas_estimation, evm_version): assert c._shl(t, s) == (t << s) % (2**256) -@pytest.mark.parametrize("evm_version", list(EVM_VERSIONS.keys())) -def test_signed_shift(get_contract_with_gas_estimation, evm_version): +def test_signed_shift(get_contract_with_gas_estimation): code = """ @external def _sar(x: int256, y: uint256) -> int256: @@ -66,7 +62,7 @@ def _sar(x: int256, y: uint256) -> int256: def _shl(x: int256, y: uint256) -> int256: return x << y """ - c = get_contract_with_gas_estimation(code, evm_version=evm_version) + c = get_contract_with_gas_estimation(code) x = 126416208461208640982146408124 y = 7128468721412412459 cases = [x, y, -x, -y] @@ -97,8 +93,7 @@ def baz(a: uint256, b: uint256, c: uint256) -> (uint256, uint256): assert tuple(c.baz(1, 6, 14)) == (1 + 8 | ~6 & 14 * 2, (1 + 8 | ~6) & 14 * 2) == (25, 24) -@pytest.mark.parametrize("evm_version", list(EVM_VERSIONS)) -def test_literals(get_contract, evm_version): +def test_literals(get_contract): code = """ @external def _shr(x: uint256) -> uint256: @@ -109,7 +104,7 @@ def _shl(x: uint256) -> uint256: return x << 3 """ - c = get_contract(code, evm_version=evm_version) + c = get_contract(code) assert c._shr(80) == 10 assert c._shl(80) == 640 diff --git a/tests/parser/syntax/test_codehash.py b/tests/parser/syntax/test_codehash.py index 6914344e27..5074d14636 100644 --- a/tests/parser/syntax/test_codehash.py +++ b/tests/parser/syntax/test_codehash.py @@ -33,7 +33,7 @@ def foo4() -> bytes32: return self.a.codehash """ settings = Settings(evm_version=evm_version, optimize=optimize) - compiled = compile_code(code, ["bytecode_runtime"], settings) + compiled = compile_code(code, ["bytecode_runtime"], settings=settings) bytecode = bytes.fromhex(compiled["bytecode_runtime"][2:]) hash_ = keccak256(bytecode) From 524c50f75197ce8588bc94693dcb8c7770588bce Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 10 Jul 2023 18:43:59 +0000 Subject: [PATCH 24/49] fix lint --- tests/parser/syntax/test_address_code.py | 2 +- vyper/ast/nodes.pyi | 3 ++- vyper/ast/utils.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/parser/syntax/test_address_code.py b/tests/parser/syntax/test_address_code.py index ff286bbdae..70ba5cbbf7 100644 --- a/tests/parser/syntax/test_address_code.py +++ b/tests/parser/syntax/test_address_code.py @@ -175,7 +175,7 @@ def code_runtime() -> Bytes[32]: return slice(self.code, 0, 32) """ contract = get_contract(code) - settings=Settings(optimize=optimize) + settings = Settings(optimize=optimize) code_compiled = compiler.compile_code( code, output_formats=["bytecode", "bytecode_runtime"], settings=settings ) diff --git a/vyper/ast/nodes.pyi b/vyper/ast/nodes.pyi index aefb1aed51..0d59a2fa63 100644 --- a/vyper/ast/nodes.pyi +++ b/vyper/ast/nodes.pyi @@ -3,7 +3,8 @@ from typing import Any, Optional, Sequence, Type, Union from .natspec import parse_natspec as parse_natspec from .utils import ast_to_dict as ast_to_dict -from .utils import parse_to_ast as parse_to_ast, parse_to_ast_with_settings as parse_to_ast_with_settings +from .utils import parse_to_ast as parse_to_ast +from .utils import parse_to_ast_with_settings as parse_to_ast_with_settings NODE_BASE_ATTRIBUTES: Any NODE_SRC_ATTRIBUTES: Any diff --git a/vyper/ast/utils.py b/vyper/ast/utils.py index 949946802f..4e669385ab 100644 --- a/vyper/ast/utils.py +++ b/vyper/ast/utils.py @@ -1,5 +1,5 @@ import ast as python_ast -from typing import Dict, List, Optional, Union, Any +from typing import Any, Dict, List, Optional, Union from vyper.ast import nodes as vy_ast from vyper.ast.annotation import annotate_python_ast From a6caacf6c5f5127060ba402a3947ea0ab8970745 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 10 Jul 2023 18:49:30 +0000 Subject: [PATCH 25/49] relax a test it passes in no-opt mode now as well --- tests/parser/types/test_dynamic_array.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/parser/types/test_dynamic_array.py b/tests/parser/types/test_dynamic_array.py index cb55c42870..17c45fd910 100644 --- a/tests/parser/types/test_dynamic_array.py +++ b/tests/parser/types/test_dynamic_array.py @@ -1584,14 +1584,9 @@ def bar2() -> uint256: newFoo.b1[1][0][0].a1[0][1][1] + \\ newFoo.b1[0][1][0].a1[0][0][0] """ - - if no_optimize: - # fails at assembly stage with too many stack variables - assert_compile_failed(lambda: get_contract(code), Exception) - else: - c = get_contract(code) - assert c.bar() == [[[3, 7], [7, 3]], [[7, 3], [0, 0]]] - assert c.bar2() == 0 + c = get_contract(code) + assert c.bar() == [[[3, 7], [7, 3]], [[7, 3], [0, 0]]] + assert c.bar2() == 0 def test_tuple_of_lists(get_contract): From 6084de48ef4205948f7b14e43e87fbf56fd48dd1 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 10 Jul 2023 15:00:43 -0400 Subject: [PATCH 26/49] raise instead of warning --- vyper/compiler/phases.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/vyper/compiler/phases.py b/vyper/compiler/phases.py index 7859eae0d4..e58957e64e 100644 --- a/vyper/compiler/phases.py +++ b/vyper/compiler/phases.py @@ -12,6 +12,7 @@ from vyper.semantics import set_data_positions, validate_semantics from vyper.semantics.types.function import ContractFunctionT from vyper.typing import InterfaceImports, StorageLayout +from vyper.exceptions import StructureException class CompilerData: @@ -96,22 +97,18 @@ def _generate_ast(self): self.settings.evm_version is not None and self.settings.evm_version != settings.evm_version ): - # TODO: consider raising an exception - warnings.warn( + raise StructureException( f"compiler settings indicate evm version {self.settings.evm_version}, " - f"but source pragma indicates {settings.evm_version}.\n" - f"using `evm version: {self.settings.evm_version}`!" + f"but source pragma indicates {settings.evm_version}." ) self.settings.evm_version = settings.evm_version if settings.optimize is not None: if self.settings.optimize is not None and self.settings.optimize != settings.optimize: - # TODO: consider raising an exception - warnings.warn( + raise StructureException( f"compiler options indicate optimization mode {self.settings.optimize}, " - f"but source pragma indicates {settings.optimize}.\n" - f"using `optimize: {self.settings.optimize}`!" + f"but source pragma indicates {settings.optimize}." ) self.settings.optimize = settings.optimize From 3091ae3202475ffdf3f942aaef1792f98ee100a9 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 10 Jul 2023 15:01:04 -0400 Subject: [PATCH 27/49] fix lint --- vyper/compiler/phases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vyper/compiler/phases.py b/vyper/compiler/phases.py index e58957e64e..0b40f49321 100644 --- a/vyper/compiler/phases.py +++ b/vyper/compiler/phases.py @@ -8,11 +8,11 @@ from vyper.codegen.global_context import GlobalContext from vyper.codegen.ir_node import IRnode from vyper.compiler.settings import OptimizationLevel, Settings +from vyper.exceptions import StructureException from vyper.ir import compile_ir, optimizer from vyper.semantics import set_data_positions, validate_semantics from vyper.semantics.types.function import ContractFunctionT from vyper.typing import InterfaceImports, StorageLayout -from vyper.exceptions import StructureException class CompilerData: From 3cb1d5c0bf31f689184477e89c546bbd444fce41 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 10 Jul 2023 15:12:59 -0400 Subject: [PATCH 28/49] update mypy needed to update to 0.940 anyways to handle match/case --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4cacaeca18..36a138aacd 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ "flake8-bugbear==20.1.4", "flake8-use-fstring==1.1", "isort==5.9.3", - "mypy==0.940", + "mypy==0.982", ], "docs": ["recommonmark", "sphinx>=6.0,<7.0", "sphinx_rtd_theme>=1.2,<1.3"], "dev": ["ipython", "pre-commit", "pyinstaller", "twine"], From 6658dae9cb12818559be5daa3a636a43e3f6447f Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 10 Jul 2023 15:14:16 -0400 Subject: [PATCH 29/49] use `OptimizationLevel.default()` in some places --- vyper/cli/vyper_json.py | 2 +- vyper/compiler/__init__.py | 4 ++-- vyper/compiler/phases.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vyper/cli/vyper_json.py b/vyper/cli/vyper_json.py index 07883fec8e..4a1c91550e 100755 --- a/vyper/cli/vyper_json.py +++ b/vyper/cli/vyper_json.py @@ -372,7 +372,7 @@ def compile_from_input_dict( warnings.warn( "optimize: is deprecated! please use one of 'gas', 'codesize', 'none'." ) - optimize = OptimizationLevel.GAS if optimize else OptimizationLevel.NONE + optimize = OptimizationLevel.default() if optimize else OptimizationLevel.NONE elif isinstance(optimize, str): optimize = OptimizationLevel.from_string(optimize) else: diff --git a/vyper/compiler/__init__.py b/vyper/compiler/__init__.py index ddcc560d1d..0b3c0d8191 100644 --- a/vyper/compiler/__init__.py +++ b/vyper/compiler/__init__.py @@ -171,8 +171,8 @@ def compile_code( evm_version: str, optional The target EVM ruleset to compile for. If not given, defaults to the latest implemented ruleset. - optimize: OptimizationLevel, optional - Set optimization mode. Defaults to OptimizationLevel.GAS + settings: Settings, optional + Compiler settings. show_gas_estimates: bool, optional Show gas estimates for abi and ir output modes interface_codes: Dict, optional diff --git a/vyper/compiler/phases.py b/vyper/compiler/phases.py index 0b40f49321..99465809bd 100644 --- a/vyper/compiler/phases.py +++ b/vyper/compiler/phases.py @@ -309,7 +309,7 @@ def generate_assembly(ir_nodes: IRnode, optimize: Optional[OptimizationLevel] = list List of assembly instructions. """ - optimize = optimize or OptimizationLevel.GAS + optimize = optimize or OptimizationLevel.default() assembly = compile_ir.compile_to_assembly(ir_nodes, optimize=optimize) if _find_nested_opcode(assembly, "DEBUG"): From 48c2611c2fb0c179e64956e9cfc0f0bfca5e9032 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 10 Jul 2023 15:35:42 -0400 Subject: [PATCH 30/49] fix no-optimize tests --- tests/conftest.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b083e00280..9c9c4191b9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,7 +41,7 @@ def pytest_addoption(parser): "--optimize", choices=["codesize", "gas", "none"], default="gas", - help="disable asm and IR optimizations", + help="change optimization mode", ) diff --git a/tox.ini b/tox.ini index 81ff7a3397..9b63630f58 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ envlist = usedevelop = True commands = core: pytest -m "not fuzzing" --showlocals {posargs:tests/} - no-opt: pytest -m "not fuzzing" --showlocals --no-optimize {posargs:tests/} + no-opt: pytest -m "not fuzzing" --showlocals --optimize none {posargs:tests/} codesize: pytest -m "not fuzzing" --showlocals --optimize codesize {posargs:tests/} basepython = py310: python3.10 From 1d3dc48bd6abc52e15df14159fa5e25b63c2a6a5 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 10 Jul 2023 15:39:11 -0400 Subject: [PATCH 31/49] fix a comment --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 36ad477d0e..b6399b3ae9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,7 +79,7 @@ jobs: strategy: matrix: python-version: [["3.10", "310"], ["3.11", "311"]] - # run in default (optimized, gas) and --no-optimize mode + # run in modes: --optimize [gas, none, codesize] flag: ["core", "no-opt", "codesize"] name: py${{ matrix.python-version[1] }}-${{ matrix.flag }} From 214274d0f8a5d90e9c646a8f1e9529f8297e3fd8 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Mon, 10 Jul 2023 17:30:03 -0400 Subject: [PATCH 32/49] add some tests for new pragma directives --- tests/ast/test_pre_parser.py | 77 ++++++++++++++++++++++++++++++- tests/compiler/test_pre_parser.py | 61 ++++++++++++++++++++++-- 2 files changed, 133 insertions(+), 5 deletions(-) diff --git a/tests/ast/test_pre_parser.py b/tests/ast/test_pre_parser.py index b7fd823686..150ee55edf 100644 --- a/tests/ast/test_pre_parser.py +++ b/tests/ast/test_pre_parser.py @@ -1,6 +1,7 @@ import pytest -from vyper.ast.pre_parser import validate_version_pragma +from vyper.ast.pre_parser import pre_parse, validate_version_pragma +from vyper.compiler.settings import OptimizationLevel, Settings from vyper.exceptions import VersionException SRC_LINE = (1, 0) # Dummy source line @@ -106,3 +107,77 @@ def test_prerelease_invalid_version_pragma(file_version, mock_version): mock_version(PRERELEASE_COMPILER_VERSION) with pytest.raises(VersionException): validate_version_pragma(file_version, (SRC_LINE)) + + +pragma_examples = [ + ( + """ + """, + Settings(), + ), + ( + """ + #pragma optimize codesize + """, + Settings(optimize=OptimizationLevel.CODESIZE), + ), + ( + """ + #pragma optimize none + """, + Settings(optimize=OptimizationLevel.NONE), + ), + ( + """ + #pragma optimize gas + """, + Settings(optimize=OptimizationLevel.GAS), + ), + ( + """ + #pragma version 0.3.10 + """, + Settings(compiler_version="0.3.10"), + ), + ( + """ + #pragma evm-version shanghai + """, + Settings(evm_version="shanghai"), + ), + ( + """ + #pragma optimize codesize + #pragma evm-version shanghai + """, + Settings(evm_version="shanghai", optimize=OptimizationLevel.GAS), + ), + ( + """ + #pragma version 0.3.10 + #pragma evm-version shanghai + """, + Settings(evm_version="shanghai", compiler_version="0.3.10"), + ), + ( + """ + #pragma version 0.3.10 + #pragma optimize gas + """, + Settings(compiler_version="0.3.10", optimize=OptimizationLevel.GAS), + ), + ( + """ + #pragma version 0.3.10 + #pragma evm-version shanghai + #pragma optimize gas + """, + Settings(compiler_version="0.3.10", optimize=OptimizationLevel.GAS, evm_version="shanghai"), + ), +] + + +@pytest.mark.parametrize("code, expected_pragmas", pragma_examples) +def parse_pragmas(code, expected_pragmas): + pragmas, _, _ = pre_parse(code) + assert pragmas == expected_pragmas diff --git a/tests/compiler/test_pre_parser.py b/tests/compiler/test_pre_parser.py index 4b747bb7d1..1761e74bad 100644 --- a/tests/compiler/test_pre_parser.py +++ b/tests/compiler/test_pre_parser.py @@ -1,6 +1,8 @@ -from pytest import raises +import pytest -from vyper.exceptions import SyntaxException +from vyper.compiler import compile_code +from vyper.compiler.settings import OptimizationLevel, Settings +from vyper.exceptions import StructureException, SyntaxException def test_semicolon_prohibited(get_contract): @@ -10,7 +12,7 @@ def test() -> int128: return a + b """ - with raises(SyntaxException): + with pytest.raises(SyntaxException): get_contract(code) @@ -70,6 +72,57 @@ def test(): assert get_contract(code) +def test_version_pragma2(get_contract): + # new, `#pragma` way of doing things + from vyper import __version__ + + installed_version = ".".join(__version__.split(".")[:3]) + + code = f""" +#pragma version {installed_version} + +@external +def test(): + pass + """ + assert get_contract(code) + + +def test_evm_version_check(assert_compile_failed): + code = """ +#pragma evm-version berlin + """ + assert compile_code(code, settings=Settings(evm_version=None)) is not None + assert compile_code(code, settings=Settings(evm_version="berlin")) is not None + # should fail if compile options indicate different evm version + # from source pragma + with pytest.raises(StructureException): + compile_code(code, settings=Settings(evm_version="shanghai")) + + +def test_optimization_mode_check(): + code = """ +#pragma optimize codesize + """ + assert compile_code(code, settings=Settings(optimize=None)) + # should fail if compile options indicate different optimization mode + # from source pragma + with pytest.raises(StructureException): + compile_code(code, settings=Settings(optimize=OptimizationLevel.GAS)) + with pytest.raises(StructureException): + compile_code(code, settings=Settings(optimize=OptimizationLevel.NONE)) + + +def test_optimization_mode_check_none(): + code = """ +#pragma optimize none + """ + assert compile_code(code, settings=Settings(optimize=None)) + # "none" conflicts with "gas" + with pytest.raises(StructureException): + compile_code(code, settings=Settings(optimize=OptimizationLevel.GAS)) + + def test_version_empty_version(assert_compile_failed, get_contract): code = """ #@version @@ -110,5 +163,5 @@ def foo(): convert( """ - with raises(SyntaxException): + with pytest.raises(SyntaxException): get_contract(code) From a102aca056c1fce502c2e0c5d8612c8e9abd1d5f Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 11 Jul 2023 06:46:41 -0400 Subject: [PATCH 33/49] fix test_grammar.py --- tests/grammar/test_grammar.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/grammar/test_grammar.py b/tests/grammar/test_grammar.py index 7e220b58ae..22aafdc0da 100644 --- a/tests/grammar/test_grammar.py +++ b/tests/grammar/test_grammar.py @@ -106,5 +106,7 @@ def has_no_docstrings(c): @hypothesis.settings(deadline=400, max_examples=500, suppress_health_check=(HealthCheck.too_slow,)) def test_grammar_bruteforce(code): if utf8_encodable(code): - tree = parse_to_ast(pre_parse(code + "\n")[1]) + + _, _, reformatted_code = pre_parse(code + "\n") + tree = parse_to_ast(reformatted_code) assert isinstance(tree, Module) From 01910df7174ac64d063b8136335aab7781461e78 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 11 Jul 2023 07:42:43 -0400 Subject: [PATCH 34/49] update docs --- docs/compiling-a-contract.rst | 22 +++++++++++++++--- docs/structure-of-a-contract.rst | 39 ++++++++++++++++++++++++++++---- 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/docs/compiling-a-contract.rst b/docs/compiling-a-contract.rst index bb5c4db3d0..90d7565ca6 100644 --- a/docs/compiling-a-contract.rst +++ b/docs/compiling-a-contract.rst @@ -99,6 +99,11 @@ See :ref:`searching_for_imports` for more information on Vyper's import system. Online Compilers ================ +Try VyperLang! +----------------- + +`Try VyperLang! `_ is a JupterHub instance hosted by the Vyper team as a sandbox for developing and testing contracts in Vyper. It requires github for login, and supports deployment via the browser. + Remix IDE --------- @@ -109,22 +114,33 @@ Remix IDE While the Vyper version of the Remix IDE compiler is updated on a regular basis, it might be a bit behind the latest version found in the master branch of the repository. Make sure the byte code matches the output from your local compiler. +.. _evm-version: + Setting the Target EVM Version ============================== -When you compile your contract code, you can specify the Ethereum Virtual Machine version to compile for, to avoid particular features or behaviours. +When you compile your contract code, you can specify the target Ethereum Virtual Machine version to compile for, to access or avoid particular features. You can specify the version either with a source code pragma or as a compiler option. It is recommended to use the compiler option when you want flexibility (for instance, ease of deploying across different chains), and the source code pragma when you want bytecode reproducibility (for instance, when verifying code on a block explorer). + +.. note:: + If the evm version specified by the compiler options conflicts with the source code pragma, an exception will be raised. + +For instance, the adding the following pragma to a contract indicates that it should be compiled for the "shanghai" fork of the EVM. + +.. code-block:: python + + #pragma evm-version shanghai .. warning:: Compiling for the wrong EVM version can result in wrong, strange and failing behaviour. Please ensure, especially if running a private chain, that you use matching EVM versions. -When compiling via ``vyper``, include the ``--evm-version`` flag: +When compiling via ``vyper``, you can specify the EVM version option using the ``--evm-version`` flag: :: $ vyper --evm-version [VERSION] -When using the JSON interface, include the ``"evmVersion"`` key within the ``"settings"`` field: +When using the JSON interface, you can include the ``"evmVersion"`` key within the ``"settings"`` field: .. code-block:: javascript diff --git a/docs/structure-of-a-contract.rst b/docs/structure-of-a-contract.rst index 8eb2c1da78..9012bec5aa 100644 --- a/docs/structure-of-a-contract.rst +++ b/docs/structure-of-a-contract.rst @@ -9,16 +9,47 @@ This section provides a quick overview of the types of data present within a con .. _structure-versions: -Version Pragma +Pragmas ============== -Vyper supports a version pragma to ensure that a contract is only compiled by the intended compiler version, or range of versions. Version strings use `NPM `_ style syntax. +Vyper supports several directives to control compiler modes and help with build reproducibility + +Version Pragma +-------------- + +The version pragma ensures that a contract is only compiled by the intended compiler version, or range of versions. Version strings use `NPM `_ style syntax. + +As of 0.3.10, the recommended way to specify the version pragma is as follows: .. code-block:: python - # @version ^0.2.0 + #pragma version ^0.3.0 + +The following declaration is equivalent, and, prior to 0.3.10, was the only supported method to specify the compiler version: + +.. code-block:: python + + # @version ^0.3.0 + + +In the above examples, the contract will only compile with Vyper versions ``0.3.x``. + +Optimization Mode +----------------- + +The optimization mode can be one of "none", "codesize", or "gas" (default). For instance, the following contract will be compiled in a way which tries to minimize codesize: + +.. code-block:: python + + #pragma optimize codesize + +The optimization mode can also be set as a compiler option. If the compiler option conflicts with the source code pragma, an exception will be raised and compilation will not continue. + +EVM Version +----------------- + +The EVM version can be set with the ``evm-version`` pragma, which is documented in :ref:`evm-version`. -In the above example, the contract only compiles with Vyper versions ``0.2.x``. .. _structure-state-variables: From 38951d39d024e65391465d1dcf42ddebac75e1c9 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 11 Jul 2023 07:44:21 -0400 Subject: [PATCH 35/49] update docs --- docs/compiling-a-contract.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/compiling-a-contract.rst b/docs/compiling-a-contract.rst index 90d7565ca6..8620eb2b04 100644 --- a/docs/compiling-a-contract.rst +++ b/docs/compiling-a-contract.rst @@ -132,7 +132,7 @@ For instance, the adding the following pragma to a contract indicates that it sh .. warning:: - Compiling for the wrong EVM version can result in wrong, strange and failing behaviour. Please ensure, especially if running a private chain, that you use matching EVM versions. + Compiling for the wrong EVM version can result in wrong, strange, or failing behavior. Please ensure, especially if running a private chain, that you use matching EVM versions. When compiling via ``vyper``, you can specify the EVM version option using the ``--evm-version`` flag: From a3bc3c23ee27573ac44fabf0915f3e70aee6145a Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 11 Jul 2023 07:47:18 -0400 Subject: [PATCH 36/49] docs: formatting --- docs/compiling-a-contract.rst | 4 ++-- docs/structure-of-a-contract.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/compiling-a-contract.rst b/docs/compiling-a-contract.rst index 8620eb2b04..208771a5a9 100644 --- a/docs/compiling-a-contract.rst +++ b/docs/compiling-a-contract.rst @@ -122,7 +122,7 @@ Setting the Target EVM Version When you compile your contract code, you can specify the target Ethereum Virtual Machine version to compile for, to access or avoid particular features. You can specify the version either with a source code pragma or as a compiler option. It is recommended to use the compiler option when you want flexibility (for instance, ease of deploying across different chains), and the source code pragma when you want bytecode reproducibility (for instance, when verifying code on a block explorer). .. note:: - If the evm version specified by the compiler options conflicts with the source code pragma, an exception will be raised. + If the evm version specified by the compiler options conflicts with the source code pragma, an exception will be raised and compilation will not continue. For instance, the adding the following pragma to a contract indicates that it should be compiled for the "shanghai" fork of the EVM. @@ -134,7 +134,7 @@ For instance, the adding the following pragma to a contract indicates that it sh Compiling for the wrong EVM version can result in wrong, strange, or failing behavior. Please ensure, especially if running a private chain, that you use matching EVM versions. -When compiling via ``vyper``, you can specify the EVM version option using the ``--evm-version`` flag: +When compiling via the ``vyper`` CLI, you can specify the EVM version option using the ``--evm-version`` flag: :: diff --git a/docs/structure-of-a-contract.rst b/docs/structure-of-a-contract.rst index 9012bec5aa..c7abb3e645 100644 --- a/docs/structure-of-a-contract.rst +++ b/docs/structure-of-a-contract.rst @@ -12,7 +12,7 @@ This section provides a quick overview of the types of data present within a con Pragmas ============== -Vyper supports several directives to control compiler modes and help with build reproducibility +Vyper supports several source code directives to control compiler modes and help with build reproducibility. Version Pragma -------------- @@ -37,7 +37,7 @@ In the above examples, the contract will only compile with Vyper versions ``0.3. Optimization Mode ----------------- -The optimization mode can be one of "none", "codesize", or "gas" (default). For instance, the following contract will be compiled in a way which tries to minimize codesize: +The optimization mode can be one of ``"none"``, ``"codesize"``, or ``"gas"`` (default). For instance, the following contract will be compiled in a way which tries to minimize codesize: .. code-block:: python From 11e678b7729658f3071c54ad82bea719aea1d2c9 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 11 Jul 2023 07:51:10 -0400 Subject: [PATCH 37/49] fix lint --- tests/grammar/test_grammar.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/grammar/test_grammar.py b/tests/grammar/test_grammar.py index 22aafdc0da..d665ca2544 100644 --- a/tests/grammar/test_grammar.py +++ b/tests/grammar/test_grammar.py @@ -106,7 +106,6 @@ def has_no_docstrings(c): @hypothesis.settings(deadline=400, max_examples=500, suppress_health_check=(HealthCheck.too_slow,)) def test_grammar_bruteforce(code): if utf8_encodable(code): - _, _, reformatted_code = pre_parse(code + "\n") tree = parse_to_ast(reformatted_code) assert isinstance(tree, Module) From dc261e188bedee65b7c0845c0ef3cfffd2e91b9a Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 11 Jul 2023 08:18:42 -0400 Subject: [PATCH 38/49] improve batch copy heuristic depending on opt mode --- vyper/codegen/core.py | 58 +++++++++++++++++++++++++++++++++++----- vyper/compiler/phases.py | 8 ++++-- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/vyper/codegen/core.py b/vyper/codegen/core.py index a330059ec7..7b5016fa21 100644 --- a/vyper/codegen/core.py +++ b/vyper/codegen/core.py @@ -1,5 +1,9 @@ +import contextlib +from typing import Generator + from vyper import ast as vy_ast from vyper.codegen.ir_node import Encoding, IRnode +from vyper.compiler.settings import OptimizationLevel from vyper.evm.address_space import CALLDATA, DATA, IMMUTABLES, MEMORY, STORAGE, TRANSIENT from vyper.evm.opcodes import version_check from vyper.exceptions import CompilerPanic, StructureException, TypeCheckFailure, TypeMismatch @@ -878,6 +882,38 @@ def make_setter(left, right): return _complex_make_setter(left, right) +_opt_level = OptimizationLevel.GAS + + +@contextlib.contextmanager +def anchor_opt_level(new_level: OptimizationLevel) -> Generator: + """ + Set the global optimization level variable for the duration of this + context manager. + """ + assert isinstance(new_level, OptimizationLevel) + + global _opt_level + try: + tmp = _opt_level + _opt_level = new_level + yield + finally: + _opt_level = tmp + + +def _opt_codesize(): + return _opt_level == OptimizationLevel.CODESIZE + + +def _opt_gas(): + return _opt_level == OptimizationLevel.GAS + + +def _opt_none(): + return _opt_level == OptimizationLevel.NONE + + def _complex_make_setter(left, right): if right.value == "~empty" and left.location == MEMORY: # optimized memzero @@ -894,23 +930,33 @@ def _complex_make_setter(left, right): keys = left.typ.tuple_keys() if left.is_pointer and right.is_pointer and right.encoding == Encoding.VYPER: + # both left and right are pointers, see if we want to batch copy + # instead of unrolling the loop. assert left.encoding == Encoding.VYPER len_ = left.typ.memory_bytes_required has_storage = STORAGE in (left.location, right.location) if has_storage: - # TODO: make this smarter, probably want to even loop for storage - # above a certain threshold. note a single sstore(dst (sload src)) - # is 8 bytes, whereas loop overhead is 17 bytes. - should_batch_copy = False + if _opt_codesize(): + # note a single sstore(dst (sload src)) is 8 bytes, + # sstore(add (dst ofst), (sload (add (src ofst)))) is 16 bytes, + # whereas loop overhead is 17 bytes. + should_batch_copy = len_ >= 32 * 3 + elif _opt_gas(): + # kind of arbitrary, but cut off when code used > ~160 bytes + should_batch_copy = len_ >= 32 * 10 + else: + # don't care, just generate the most readable version + should_batch_copy = True else: # 10 words is the cutoff for memory copy where identity is cheaper # than unrolled mloads/mstores # if MCOPY is available, mcopy is *always* better (except in # the 1 word case, but that is already handled by copy_bytes). - if right.location == MEMORY: + if right.location == MEMORY and _opt_gas(): should_batch_copy = len_ >= 32 * 10 or version_check(begin="cancun") - # calldata or code to memory - batch copy is always better. + # calldata to memory, code to memory, or prioritize codesize - + # batch copy is always better. else: should_batch_copy = True diff --git a/vyper/compiler/phases.py b/vyper/compiler/phases.py index 99465809bd..4e1bd9e6c3 100644 --- a/vyper/compiler/phases.py +++ b/vyper/compiler/phases.py @@ -5,6 +5,7 @@ from vyper import ast as vy_ast from vyper.codegen import module +from vyper.codegen.core import anchor_opt_level from vyper.codegen.global_context import GlobalContext from vyper.codegen.ir_node import IRnode from vyper.compiler.settings import OptimizationLevel, Settings @@ -268,7 +269,9 @@ def generate_folded_ast( return vyper_module_folded, symbol_tables -def generate_ir_nodes(global_ctx: GlobalContext, optimize: bool) -> tuple[IRnode, IRnode]: +def generate_ir_nodes( + global_ctx: GlobalContext, optimize: OptimizationLevel +) -> tuple[IRnode, IRnode]: """ Generate the intermediate representation (IR) from the contextualized AST. @@ -288,7 +291,8 @@ def generate_ir_nodes(global_ctx: GlobalContext, optimize: bool) -> tuple[IRnode IR to generate deployment bytecode IR to generate runtime bytecode """ - ir_nodes, ir_runtime = module.generate_ir_for_module(global_ctx) + with anchor_opt_level(optimize): + ir_nodes, ir_runtime = module.generate_ir_for_module(global_ctx) if optimize != OptimizationLevel.NONE: ir_nodes = optimizer.optimize(ir_nodes) ir_runtime = optimizer.optimize(ir_runtime) From 90a65fc7585be8fc225d6b70dd328cf25d4fd29d Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 11 Jul 2023 10:29:40 -0400 Subject: [PATCH 39/49] move slice tests from `fuzzing` to `not fuzzing` to ensure testing across optimization modes --- setup.cfg | 1 - tests/parser/functions/test_slice.py | 78 +++++++++++++----------- tests/parser/types/test_dynamic_array.py | 1 - 3 files changed, 41 insertions(+), 39 deletions(-) diff --git a/setup.cfg b/setup.cfg index d18ffe2ac7..dd4a32a3ac 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,7 +31,6 @@ addopts = -n auto --cov-report html --cov-report xml --cov=vyper - --hypothesis-show-statistics python_files = test_*.py testpaths = tests markers = diff --git a/tests/parser/functions/test_slice.py b/tests/parser/functions/test_slice.py index 11d834bf42..60feb7f5ae 100644 --- a/tests/parser/functions/test_slice.py +++ b/tests/parser/functions/test_slice.py @@ -1,4 +1,6 @@ +import hypothesis.strategies as st import pytest +from hypothesis import given, settings from vyper.exceptions import ArgumentException @@ -9,14 +11,6 @@ def _generate_bytes(length): return bytes(list(range(length))) -# good numbers to try -_fun_numbers = [0, 1, 5, 31, 32, 33, 64, 99, 100, 101] - - -# [b"", b"\x01", b"\x02"...] -_bytes_examples = [_generate_bytes(i) for i in _fun_numbers if i <= 100] - - def test_basic_slice(get_contract_with_gas_estimation): code = """ @external @@ -31,12 +25,16 @@ def slice_tower_test(inp1: Bytes[50]) -> Bytes[50]: assert x == b"klmnopqrst", x -@pytest.mark.parametrize("bytesdata", _bytes_examples) -@pytest.mark.parametrize("start", _fun_numbers) +# note: optimization boundaries at 32, 64 and 320 depending on mode +_draw_1024 = st.integers(min_value=0, max_value=1024) +_draw_1024_1 = st.integers(min_value=1, max_value=1024) +_bytes_1024 = st.binary(min_size=0, max_size=1024) + + @pytest.mark.parametrize("literal_start", (True, False)) -@pytest.mark.parametrize("length", _fun_numbers) @pytest.mark.parametrize("literal_length", (True, False)) -@pytest.mark.fuzzing +@given(start=_draw_1024, length=_draw_1024, length_bound=_draw_1024_1, bytesdata=_bytes_1024) +@settings(max_examples=100, deadline=None) def test_slice_immutable( get_contract, assert_compile_failed, @@ -46,47 +44,48 @@ def test_slice_immutable( literal_start, length, literal_length, + length_bound, ): _start = start if literal_start else "start" _length = length if literal_length else "length" code = f""" -IMMUTABLE_BYTES: immutable(Bytes[100]) -IMMUTABLE_SLICE: immutable(Bytes[100]) +IMMUTABLE_BYTES: immutable(Bytes[{length_bound}]) +IMMUTABLE_SLICE: immutable(Bytes[{length_bound}]) @external -def __init__(inp: Bytes[100], start: uint256, length: uint256): +def __init__(inp: Bytes[{length_bound}], start: uint256, length: uint256): IMMUTABLE_BYTES = inp IMMUTABLE_SLICE = slice(IMMUTABLE_BYTES, {_start}, {_length}) @external -def do_splice() -> Bytes[100]: +def do_splice() -> Bytes[{length_bound}]: return IMMUTABLE_SLICE """ + def _get_contract(): + return get_contract(code, bytesdata, start, length) + if ( - (start + length > 100 and literal_start and literal_length) - or (literal_length and length > 100) - or (literal_start and start > 100) + (start + length > length_bound and literal_start and literal_length) + or (literal_length and length > length_bound) + or (literal_start and start > length_bound) or (literal_length and length < 1) + or (len(bytesdata) > length_bound) ): - assert_compile_failed( - lambda: get_contract(code, bytesdata, start, length), ArgumentException - ) + assert_compile_failed(lambda: _get_contract(), ArgumentException) elif start + length > len(bytesdata): - assert_tx_failed(lambda: get_contract(code, bytesdata, start, length)) + assert_tx_failed(lambda: _get_contract()) else: - c = get_contract(code, bytesdata, start, length) + c = _get_contract() assert c.do_splice() == bytesdata[start : start + length] @pytest.mark.parametrize("location", ("storage", "calldata", "memory", "literal", "code")) -@pytest.mark.parametrize("bytesdata", _bytes_examples) -@pytest.mark.parametrize("start", _fun_numbers) @pytest.mark.parametrize("literal_start", (True, False)) -@pytest.mark.parametrize("length", _fun_numbers) @pytest.mark.parametrize("literal_length", (True, False)) -@pytest.mark.fuzzing +@given(start=_draw_1024, length=_draw_1024, length_bound=_draw_1024_1, bytesdata=_bytes_1024) +@settings(max_examples=100, deadline=None) def test_slice_bytes( get_contract, assert_compile_failed, @@ -97,9 +96,10 @@ def test_slice_bytes( literal_start, length, literal_length, + length_bound, ): if location == "memory": - spliced_code = "foo: Bytes[100] = inp" + spliced_code = f"foo: Bytes[{length_bound}] = inp" foo = "foo" elif location == "storage": spliced_code = "self.foo = inp" @@ -120,31 +120,35 @@ def test_slice_bytes( _length = length if literal_length else "length" code = f""" -foo: Bytes[100] -IMMUTABLE_BYTES: immutable(Bytes[100]) +foo: Bytes[{length_bound}] +IMMUTABLE_BYTES: immutable(Bytes[{length_bound}]) @external -def __init__(foo: Bytes[100]): +def __init__(foo: Bytes[{length_bound}]): IMMUTABLE_BYTES = foo @external -def do_slice(inp: Bytes[100], start: uint256, length: uint256) -> Bytes[100]: +def do_slice(inp: Bytes[{length_bound}], start: uint256, length: uint256) -> Bytes[{length_bound}]: {spliced_code} return slice({foo}, {_start}, {_length}) """ - length_bound = len(bytesdata) if location == "literal" else 100 + def _get_contract(): + return get_contract(code, bytesdata) + + length_bound = len(bytesdata) if location == "literal" else length_bound if ( (start + length > length_bound and literal_start and literal_length) or (literal_length and length > length_bound) or (literal_start and start > length_bound) or (literal_length and length < 1) + or len(bytesdata) > length_bound ): - assert_compile_failed(lambda: get_contract(code, bytesdata), ArgumentException) + assert_compile_failed(lambda: _get_contract(), ArgumentException) elif start + length > len(bytesdata): - c = get_contract(code, bytesdata) + c = _get_contract() assert_tx_failed(lambda: c.do_slice(bytesdata, start, length)) else: - c = get_contract(code, bytesdata) + c = _get_contract() assert c.do_slice(bytesdata, start, length) == bytesdata[start : start + length], code diff --git a/tests/parser/types/test_dynamic_array.py b/tests/parser/types/test_dynamic_array.py index 567ef658a4..9231d1979f 100644 --- a/tests/parser/types/test_dynamic_array.py +++ b/tests/parser/types/test_dynamic_array.py @@ -2,7 +2,6 @@ import pytest -from vyper.compiler.settings import OptimizationLevel from vyper.exceptions import ( ArgumentException, ArrayIndexException, From cbd2aedd8f3744acf985df21461829078d8734b9 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 11 Jul 2023 11:32:29 -0400 Subject: [PATCH 40/49] reduce num examples --- tests/parser/functions/test_slice.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parser/functions/test_slice.py b/tests/parser/functions/test_slice.py index 60feb7f5ae..da161a0a28 100644 --- a/tests/parser/functions/test_slice.py +++ b/tests/parser/functions/test_slice.py @@ -34,7 +34,7 @@ def slice_tower_test(inp1: Bytes[50]) -> Bytes[50]: @pytest.mark.parametrize("literal_start", (True, False)) @pytest.mark.parametrize("literal_length", (True, False)) @given(start=_draw_1024, length=_draw_1024, length_bound=_draw_1024_1, bytesdata=_bytes_1024) -@settings(max_examples=100, deadline=None) +@settings(max_examples=25, deadline=None) def test_slice_immutable( get_contract, assert_compile_failed, @@ -85,7 +85,7 @@ def _get_contract(): @pytest.mark.parametrize("literal_start", (True, False)) @pytest.mark.parametrize("literal_length", (True, False)) @given(start=_draw_1024, length=_draw_1024, length_bound=_draw_1024_1, bytesdata=_bytes_1024) -@settings(max_examples=100, deadline=None) +@settings(max_examples=25, deadline=None) def test_slice_bytes( get_contract, assert_compile_failed, From 90656896960594bc9fdf09f29fc3d09349842227 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 11 Jul 2023 13:29:12 -0400 Subject: [PATCH 41/49] fix a slice test --- tests/parser/functions/test_slice.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/parser/functions/test_slice.py b/tests/parser/functions/test_slice.py index da161a0a28..c7ed8f022b 100644 --- a/tests/parser/functions/test_slice.py +++ b/tests/parser/functions/test_slice.py @@ -71,10 +71,10 @@ def _get_contract(): or (literal_length and length > length_bound) or (literal_start and start > length_bound) or (literal_length and length < 1) - or (len(bytesdata) > length_bound) ): assert_compile_failed(lambda: _get_contract(), ArgumentException) - elif start + length > len(bytesdata): + elif start + length > len(bytesdata) or (len(bytesdata) > length_bound): + # deploy fail assert_tx_failed(lambda: _get_contract()) else: c = _get_contract() @@ -141,9 +141,11 @@ def _get_contract(): or (literal_length and length > length_bound) or (literal_start and start > length_bound) or (literal_length and length < 1) - or len(bytesdata) > length_bound ): assert_compile_failed(lambda: _get_contract(), ArgumentException) + elif len(bytesdata) > length_bound: + # deploy fail + assert_tx_failed(lambda: _get_contract()) elif start + length > len(bytesdata): c = _get_contract() assert_tx_failed(lambda: c.do_slice(bytesdata, start, length)) From 6e70dadb8148afad6d1780d7e99860baf4186626 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 14 Jul 2023 13:49:53 -0400 Subject: [PATCH 42/49] improve cost function for storage batch copy --- vyper/codegen/core.py | 8 +++++++- vyper/codegen/ir_node.py | 16 +++++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/vyper/codegen/core.py b/vyper/codegen/core.py index 7b5016fa21..a0511295c4 100644 --- a/vyper/codegen/core.py +++ b/vyper/codegen/core.py @@ -941,7 +941,13 @@ def _complex_make_setter(left, right): # note a single sstore(dst (sload src)) is 8 bytes, # sstore(add (dst ofst), (sload (add (src ofst)))) is 16 bytes, # whereas loop overhead is 17 bytes. - should_batch_copy = len_ >= 32 * 3 + base_cost = 3 + if left._optimized.is_literal: + # code size is smaller since add is performed at compile-time + base_cost += 1 + if right._optimized.is_literal: + base_cost += 1 + should_batch_copy = len_ >= 32 * base_cost elif _opt_gas(): # kind of arbitrary, but cut off when code used > ~160 bytes should_batch_copy = len_ >= 32 * 10 diff --git a/vyper/codegen/ir_node.py b/vyper/codegen/ir_node.py index f7698fbabb..9516198691 100644 --- a/vyper/codegen/ir_node.py +++ b/vyper/codegen/ir_node.py @@ -49,10 +49,7 @@ class Encoding(Enum): # this creates a magical block which maps to IR `with` class _WithBuilder: def __init__(self, ir_node, name, should_inline=False): - # TODO figure out how to fix this circular import - from vyper.ir.optimizer import optimize - - if should_inline and optimize(ir_node).is_complex_ir: + if should_inline and ir_node._optimized.is_complex_ir: # this can only mean trouble raise CompilerPanic("trying to inline a complex IR node") @@ -366,6 +363,13 @@ def is_pointer(self): # eventually return self.location is not None + @cached_property + def _optimized(self): + # TODO figure out how to fix this circular import + from vyper.ir.optimizer import optimize + + return optimize(self) + # This function is slightly confusing but abstracts a common pattern: # when an IR value needs to be computed once and then cached as an # IR value (if it is expensive, or more importantly if its computation @@ -382,13 +386,11 @@ def is_pointer(self): # return builder.resolve(ret) # ``` def cache_when_complex(self, name): - from vyper.ir.optimizer import optimize - # for caching purposes, see if the ir_node will be optimized # because a non-literal expr could turn into a literal, # (e.g. `(add 1 2)`) # TODO this could really be moved into optimizer.py - should_inline = not optimize(self).is_complex_ir + should_inline = not self._optimized.is_complex_ir return _WithBuilder(self, name, should_inline) From e40da9160274233a6c63e284faf5aa6e3bbe87b6 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 14 Jul 2023 13:57:57 -0400 Subject: [PATCH 43/49] use property instead of cached_property --- vyper/codegen/core.py | 2 +- vyper/codegen/ir_node.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vyper/codegen/core.py b/vyper/codegen/core.py index a0511295c4..0614979063 100644 --- a/vyper/codegen/core.py +++ b/vyper/codegen/core.py @@ -940,7 +940,7 @@ def _complex_make_setter(left, right): if _opt_codesize(): # note a single sstore(dst (sload src)) is 8 bytes, # sstore(add (dst ofst), (sload (add (src ofst)))) is 16 bytes, - # whereas loop overhead is 17 bytes. + # whereas loop overhead is 16-17 bytes. base_cost = 3 if left._optimized.is_literal: # code size is smaller since add is performed at compile-time diff --git a/vyper/codegen/ir_node.py b/vyper/codegen/ir_node.py index 9516198691..0895e5f02d 100644 --- a/vyper/codegen/ir_node.py +++ b/vyper/codegen/ir_node.py @@ -363,7 +363,7 @@ def is_pointer(self): # eventually return self.location is not None - @cached_property + @property # probably could be cached_property but be paranoid def _optimized(self): # TODO figure out how to fix this circular import from vyper.ir.optimizer import optimize From b383e4106cfa31bf33ca67f41c68f834ae7e2d9d Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 14 Jul 2023 14:36:45 -0400 Subject: [PATCH 44/49] improve cost estimate for pre-cancun memory copies --- vyper/codegen/core.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/vyper/codegen/core.py b/vyper/codegen/core.py index 0614979063..e507efe3ad 100644 --- a/vyper/codegen/core.py +++ b/vyper/codegen/core.py @@ -955,13 +955,29 @@ def _complex_make_setter(left, right): # don't care, just generate the most readable version should_batch_copy = True else: - # 10 words is the cutoff for memory copy where identity is cheaper + # find a cutoff for memory copy where identity is cheaper # than unrolled mloads/mstores # if MCOPY is available, mcopy is *always* better (except in # the 1 word case, but that is already handled by copy_bytes). - if right.location == MEMORY and _opt_gas(): - should_batch_copy = len_ >= 32 * 10 or version_check(begin="cancun") - # calldata to memory, code to memory, or prioritize codesize - + if right.location == MEMORY and _opt_gas() and not version_check(begin="cancun"): + # cost for 0th word - (mstore dst (mload src)) + base_unroll_cost = 12 + nth_word_cost = base_unroll_cost + if not left._optimized.is_literal: + # (mstore (add N dst) (mload src)) + nth_word_cost += 6 + if not right._optimized.is_literal: + # (mstore dst (mload (add N src))) + nth_word_cost += 6 + + identity_base_cost = 115 # staticcall 4 gas dst len src len + + n_words = ceil32(len_) // 32 + should_batch_copy = ( + base_unroll_cost + (nth_word_cost * (n_words - 1)) >= identity_base_cost + ) + + # calldata to memory, code to memory, cancun, or codesize - # batch copy is always better. else: should_batch_copy = True From 6ac32d7c89b945dfd3b2d0dcfcf5e62c5d1e4a4c Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 14 Jul 2023 14:41:19 -0400 Subject: [PATCH 45/49] add --optimize "none" --- vyper/cli/vyper_compile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vyper/cli/vyper_compile.py b/vyper/cli/vyper_compile.py index 71e78dd666..55e0fc82b2 100755 --- a/vyper/cli/vyper_compile.py +++ b/vyper/cli/vyper_compile.py @@ -105,7 +105,7 @@ def _parse_args(argv): dest="evm_version", ) parser.add_argument("--no-optimize", help="Do not optimize", action="store_true") - parser.add_argument("--optimize", help="Optimization flag", choices=["gas", "codesize"]) + parser.add_argument("--optimize", help="Optimization flag", choices=["gas", "codesize", "none"]) parser.add_argument( "--no-bytecode-metadata", help="Do not add metadata to bytecode", action="store_true" ) From 87fb2026e5164c2059d927358bf107de3d2aae05 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 14 Jul 2023 15:08:45 -0400 Subject: [PATCH 46/49] add comments on costing --- vyper/codegen/core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vyper/codegen/core.py b/vyper/codegen/core.py index e507efe3ad..4acfe839ed 100644 --- a/vyper/codegen/core.py +++ b/vyper/codegen/core.py @@ -938,7 +938,7 @@ def _complex_make_setter(left, right): has_storage = STORAGE in (left.location, right.location) if has_storage: if _opt_codesize(): - # note a single sstore(dst (sload src)) is 8 bytes, + # assuming PUSH2, a single sstore(dst (sload src)) is 8 bytes, # sstore(add (dst ofst), (sload (add (src ofst)))) is 16 bytes, # whereas loop overhead is 16-17 bytes. base_cost = 3 @@ -947,6 +947,9 @@ def _complex_make_setter(left, right): base_cost += 1 if right._optimized.is_literal: base_cost += 1 + # the formula is a heuristic, but it works. + # (CMC 2023-07-14 could get more detailed for PUSH1 vs + # PUSH2 etc but not worried about that too much now) should_batch_copy = len_ >= 32 * base_cost elif _opt_gas(): # kind of arbitrary, but cut off when code used > ~160 bytes From ea2cbb1791326947d8832f2f8b67a4be419ddb4d Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 14 Jul 2023 15:15:15 -0400 Subject: [PATCH 47/49] update a comment --- vyper/codegen/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vyper/codegen/core.py b/vyper/codegen/core.py index 4acfe839ed..54d773dbd7 100644 --- a/vyper/codegen/core.py +++ b/vyper/codegen/core.py @@ -949,7 +949,9 @@ def _complex_make_setter(left, right): base_cost += 1 # the formula is a heuristic, but it works. # (CMC 2023-07-14 could get more detailed for PUSH1 vs - # PUSH2 etc but not worried about that too much now) + # PUSH2 etc but not worried about that too much now, + # it's probably better to add a proper unroll rule in the + # optimizer.) should_batch_copy = len_ >= 32 * base_cost elif _opt_gas(): # kind of arbitrary, but cut off when code used > ~160 bytes From bfa53aaa4c77e1b6d53b437d362b23b5df86aa06 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 14 Jul 2023 16:45:12 -0400 Subject: [PATCH 48/49] fix fuzzer test --- tests/parser/functions/test_slice.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/parser/functions/test_slice.py b/tests/parser/functions/test_slice.py index c7ed8f022b..f1b642b28d 100644 --- a/tests/parser/functions/test_slice.py +++ b/tests/parser/functions/test_slice.py @@ -135,15 +135,16 @@ def do_slice(inp: Bytes[{length_bound}], start: uint256, length: uint256) -> Byt def _get_contract(): return get_contract(code, bytesdata) - length_bound = len(bytesdata) if location == "literal" else length_bound + data_length = len(bytesdata) if location == "literal" else length_bound if ( - (start + length > length_bound and literal_start and literal_length) - or (literal_length and length > length_bound) - or (literal_start and start > length_bound) + (start + length > data_length and literal_start and literal_length) + or (literal_length and length > data_length) + or (location == "literal" and len(bytesdata) > length_bound) + or (literal_start and start > data_length) or (literal_length and length < 1) ): assert_compile_failed(lambda: _get_contract(), ArgumentException) - elif len(bytesdata) > length_bound: + elif len(bytesdata) > data_length: # deploy fail assert_tx_failed(lambda: _get_contract()) elif start + length > len(bytesdata): From 1c64f2cc9e7f11de8b5a17e068358beccefec358 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Sat, 15 Jul 2023 10:37:25 -0400 Subject: [PATCH 49/49] add a sanity check --- vyper/codegen/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vyper/codegen/core.py b/vyper/codegen/core.py index 54d773dbd7..5b16938e99 100644 --- a/vyper/codegen/core.py +++ b/vyper/codegen/core.py @@ -957,6 +957,7 @@ def _complex_make_setter(left, right): # kind of arbitrary, but cut off when code used > ~160 bytes should_batch_copy = len_ >= 32 * 10 else: + assert _opt_none() # don't care, just generate the most readable version should_batch_copy = True else: