From b48aa0661b4807696f5db0589c851d56d11ee182 Mon Sep 17 00:00:00 2001 From: jherrera-jump Date: Thu, 6 Feb 2025 19:18:05 +0000 Subject: [PATCH] pack: fix cost model calculation --- agave | 2 +- src/app/fdctl/external_functions.c | 4 +- src/app/fdctl/run/tiles/fd_bank.c | 101 ++--- src/ballet/txn/fd_txn.h | 34 +- src/disco/pack/fd_compute_budget_program.h | 28 +- src/disco/pack/fd_microblock.h | 6 +- src/disco/pack/fd_pack.c | 23 +- src/disco/pack/fd_pack.h | 17 +- src/disco/pack/fd_pack_cost.h | 150 ++++--- src/disco/pack/test_compute_budget_program.c | 38 +- src/disco/pack/test_pack.c | 403 +++++++++++-------- src/flamenco/runtime/fd_runtime.c | 3 +- src/flamenco/runtime/tests/fd_pack_test.c | 5 +- 13 files changed, 467 insertions(+), 347 deletions(-) diff --git a/agave b/agave index 327b5a1dfb..63f125908c 160000 --- a/agave +++ b/agave @@ -1 +1 @@ -Subproject commit 327b5a1dfb2a10410e87c28a81feec9d6c6f4692 +Subproject commit 63f125908cd09a3a49eab9d93ef50af8919b58d3 diff --git a/src/app/fdctl/external_functions.c b/src/app/fdctl/external_functions.c index 21cca6ea4f..234c2853e3 100644 --- a/src/app/fdctl/external_functions.c +++ b/src/app/fdctl/external_functions.c @@ -5,8 +5,8 @@ extern void fd_ext_validator_main( const char ** args FD_PARAM_UNUSED ) {} extern void fd_ext_genesis_main( const char ** args FD_PARAM_UNUSED ) {} extern void * fd_ext_bank_pre_balance_info( void const * bank FD_PARAM_UNUSED, void * txns FD_PARAM_UNUSED, ulong txn_cnt FD_PARAM_UNUSED ) { return NULL; } -extern void * fd_ext_bank_load_and_execute_txns( void const * bank FD_PARAM_UNUSED, void * txns FD_PARAM_UNUSED, ulong txn_cnt FD_PARAM_UNUSED, int * out_load_results FD_PARAM_UNUSED, int * out_executing_results FD_PARAM_UNUSED, int * out_executed_results FD_PARAM_UNUSED, uint * out_consumed_cus FD_PARAM_UNUSED ) { return NULL; } -extern int fd_ext_bank_execute_and_commit_bundle( void const * bank FD_PARAM_UNUSED, void * txns FD_PARAM_UNUSED, ulong txn_cnt FD_PARAM_UNUSED, uint * out_consumed_cus FD_PARAM_UNUSED ) { return 0; } +extern int fd_ext_bank_execute_and_commit_bundle( void const * bank FD_PARAM_UNUSED, void * txns FD_PARAM_UNUSED, ulong txn_cnt FD_PARAM_UNUSED, uint * actual_execution_cus FD_PARAM_UNUSED, uint * actual_acct_data_cus FD_PARAM_UNUSED ) { return 0; } +extern void * fd_ext_bank_load_and_execute_txns( void const * bank FD_PARAM_UNUSED, void * txns FD_PARAM_UNUSED, ulong txn_cnt FD_PARAM_UNUSED, int * out_load_results FD_PARAM_UNUSED, int * out_executing_results FD_PARAM_UNUSED, int * out_executed_results FD_PARAM_UNUSED, uint * out_consumed_exec_cus FD_PARAM_UNUSED, uint * out_consumed_acct_data_cus FD_PARAM_UNUSED ) { return NULL; } extern void fd_ext_bank_acquire( void const * bank FD_PARAM_UNUSED ) {} extern void fd_ext_bank_release( void const * bank FD_PARAM_UNUSED ) {} extern void fd_ext_bank_release_thunks( void * load_and_execute_output FD_PARAM_UNUSED ) {} diff --git a/src/app/fdctl/run/tiles/fd_bank.c b/src/app/fdctl/run/tiles/fd_bank.c index 91b5e1cc4d..164442570e 100644 --- a/src/app/fdctl/run/tiles/fd_bank.c +++ b/src/app/fdctl/run/tiles/fd_bank.c @@ -1,6 +1,7 @@ #include "../../../../disco/tiles.h" #include "../../../../disco/pack/fd_pack.h" +#include "../../../../disco/pack/fd_pack_cost.h" #include "../../../../ballet/blake3/fd_blake3.h" #include "../../../../ballet/bmtree/fd_bmtree.h" #include "../../../../disco/metrics/fd_metrics.h" @@ -95,8 +96,8 @@ before_frag( fd_bank_ctx_t * ctx, } extern void * fd_ext_bank_pre_balance_info( void const * bank, void * txns, ulong txn_cnt ); -extern void * fd_ext_bank_load_and_execute_txns( void const * bank, void * txns, ulong txn_cnt, int * out_processing_results, int * out_transaction_err, uint * out_consumed_cus ); -extern int fd_ext_bank_execute_and_commit_bundle( void const * bank, void * txns, ulong txn_cnt, uint * out_consumed_cus ); +extern int fd_ext_bank_execute_and_commit_bundle( void const * bank, void * txns, ulong txn_cnt, uint * actual_execution_cus, uint * actual_acct_data_cus ); +extern void * fd_ext_bank_load_and_execute_txns( void const * bank, void * txns, ulong txn_cnt, int * out_processing_results, int * out_transaction_err, uint * out_consumed_exec_cus, uint * out_consumed_acct_data_cus ); extern void fd_ext_bank_commit_txns( void const * bank, void const * txns, ulong txn_cnt , void * load_and_execute_output, void * pre_balance_info ); extern void fd_ext_bank_release_thunks( void * load_and_execute_output ); extern void fd_ext_bank_release_pre_balance_info( void * pre_balance_info ); @@ -186,9 +187,10 @@ handle_microblock( fd_bank_ctx_t * ctx, /* Just because a transaction was executed doesn't mean it succeeded, but all executed transactions get committed. */ - int processing_results[ MAX_TXN_PER_MICROBLOCK ] = { 0 }; - int transaction_err [ MAX_TXN_PER_MICROBLOCK ] = { 0 }; - uint consumed_cus [ MAX_TXN_PER_MICROBLOCK ] = { 0U }; + int processing_results [ MAX_TXN_PER_MICROBLOCK ] = { 0 }; + int transaction_err [ MAX_TXN_PER_MICROBLOCK ] = { 0 }; + uint consumed_exec_cus [ MAX_TXN_PER_MICROBLOCK ] = { 0U }; + uint consumed_acct_data_cus[ MAX_TXN_PER_MICROBLOCK ] = { 0U }; void * pre_balance_info = fd_ext_bank_pre_balance_info( ctx->_bank, ctx->txn_abi_mem, sanitized_txn_cnt ); @@ -197,17 +199,27 @@ handle_microblock( fd_bank_ctx_t * ctx, sanitized_txn_cnt, processing_results, transaction_err, - consumed_cus ); + consumed_exec_cus, + consumed_acct_data_cus ); ulong sanitized_idx = 0UL; for( ulong i=0UL; ipack_cu.requested_execution_cus; - uint non_execution_cus = txn->pack_cu.non_execution_cus; - /* Assume failure, set below if success. If it doesn't land in the + uint requested_exec_plus_acct_data_cus = txn->pack_cu.requested_exec_plus_acct_data_cus; + uint non_execution_cus = txn->pack_cu.non_execution_cus; + + if( FD_UNLIKELY( fd_txn_is_simple_vote_transaction( TXN(txn), txn->payload ) ) ) { + /* Simple votes are charged fixed amounts of compute regardless of + the real cost they incur. fd_ext_bank_load_and_execute_txns + returns the real cost, however, so we override it here. */ + consumed_exec_cus[i] = FD_PACK_VOTE_DEFAULT_COMPUTE_UNITS; + consumed_acct_data_cus[i] = 0; + } + + /* Assume failure, set below if success. If it doesn't land cin the block, rebate the non-execution CUs too. */ - txn->bank_cu.rebated_cus = requested_cus + non_execution_cus; + txn->bank_cu.rebated_cus = requested_exec_plus_acct_data_cus + non_execution_cus; txn->flags &= ~FD_TXN_P_FLAGS_EXECUTE_SUCCESS; if( FD_UNLIKELY( !(txn->flags & FD_TXN_P_FLAGS_SANITIZE_SUCCESS) ) ) continue; @@ -234,23 +246,16 @@ handle_microblock( fd_bank_ctx_t * ctx, if( transaction_err[ sanitized_idx-1UL ] ) ctx->metrics.exec_failed++; else ctx->metrics.success++; - uint executed_cus = consumed_cus[ sanitized_idx-1UL ]; - txn->bank_cu.actual_consumed_cus = non_execution_cus + executed_cus; - if( FD_UNLIKELY( executed_cus>requested_cus ) ) { - /* There's basically a bug in the Agave codebase right now - regarding the cost model for some transactions. Some built-in - instructions like creating an address lookup table consume more - CUs than the cost model allocates for them, which is only - allowed because the runtime computes requested CUs differently - from the cost model. Rather than implement a broken system, - we'll just permit the risk of slightly overpacking blocks by - ignoring these transactions when it comes to rebating. */ - FD_LOG_INFO(( "Transaction executed %u CUs but only requested %u CUs", executed_cus, requested_cus )); - FD_MCNT_INC( BANK, COST_MODEL_UNDERCOUNT, 1UL ); - txn->bank_cu.rebated_cus = 0U; - continue; - } - txn->bank_cu.rebated_cus = requested_cus - executed_cus; + uint actual_execution_cus = consumed_exec_cus[ sanitized_idx-1UL ]; + uint actual_acct_data_cus = consumed_acct_data_cus[ sanitized_idx-1UL ]; + txn->bank_cu.actual_consumed_cus = non_execution_cus + actual_execution_cus + actual_acct_data_cus; + + /* The VM will stop executing and fail an instruction immediately if + it exceeds its requested CUs. A transaction which requests less + account data than it actually consumes will fail in the account + loading stage. */ + FD_TEST( actual_execution_cus + actual_acct_data_cus <= requested_exec_plus_acct_data_cus ); + txn->bank_cu.rebated_cus = requested_exec_plus_acct_data_cus - ( actual_execution_cus + actual_acct_data_cus ); } /* Commit must succeed so no failure path. This function takes @@ -340,20 +345,30 @@ handle_bundle( fd_bank_ctx_t * ctx, sidecar_footprint_bytes += FD_BANK_ABI_TXN_FOOTPRINT_SIDECAR( txn1->acct_addr_cnt, txn1->addr_table_adtl_cnt, txn1->instr_cnt, txn1->addr_table_lookup_cnt ); } - uint consumed_cus[ MAX_TXN_PER_MICROBLOCK ] = { 0U }; + uint actual_execution_cus[ MAX_TXN_PER_MICROBLOCK ] = { 0U }; + uint actual_acct_data_cus[ MAX_TXN_PER_MICROBLOCK ] = { 0U }; + uint consumed_cus [ MAX_TXN_PER_MICROBLOCK ] = { 0U }; if( FD_LIKELY( execution_success ) ) { /* TODO: Plumb through errors. */ - execution_success = fd_ext_bank_execute_and_commit_bundle( ctx->_bank, ctx->txn_abi_mem, txn_cnt, consumed_cus ); + execution_success = fd_ext_bank_execute_and_commit_bundle( ctx->_bank, ctx->txn_abi_mem, txn_cnt, actual_execution_cus, actual_acct_data_cus ); } if( FD_LIKELY( execution_success ) ) { ctx->metrics.success += txn_cnt; ctx->metrics.transaction_result[ FD_METRICS_ENUM_TRANSACTION_ERROR_V_SUCCESS_IDX ] += txn_cnt; - for( ulong i=0UL; ipack_cu.requested_exec_plus_acct_data_cus; + /* Don't double count metrics for transactions that failed to sanitize. */ if( FD_UNLIKELY( !(txn->flags & FD_TXN_P_FLAGS_SANITIZE_SUCCESS) ) ) continue; @@ -371,30 +386,16 @@ handle_bundle( fd_bank_ctx_t * ctx, for( ulong i=0UL; ipack_cu.requested_execution_cus; - uint non_execution_cus = txn->pack_cu.non_execution_cus; + uint requested_exec_plus_acct_data_cus = txn->pack_cu.requested_exec_plus_acct_data_cus; + uint non_execution_cus = txn->pack_cu.non_execution_cus; /* Assume failure, set below if success. If it doesn't land in the block, rebate the non-execution CUs too. */ - txn->bank_cu.rebated_cus = requested_cus + non_execution_cus; + txn->bank_cu.rebated_cus = requested_exec_plus_acct_data_cus + non_execution_cus; if( FD_LIKELY( (txn->flags & (FD_TXN_P_FLAGS_SANITIZE_SUCCESS|FD_TXN_P_FLAGS_EXECUTE_SUCCESS))==(FD_TXN_P_FLAGS_SANITIZE_SUCCESS|FD_TXN_P_FLAGS_EXECUTE_SUCCESS) ) ) { - uint executed_cus = consumed_cus[ i ]; - txn->bank_cu.actual_consumed_cus = non_execution_cus + executed_cus; - if( FD_UNLIKELY( executed_cus>requested_cus ) ) { - /* There's basically a bug in the Agave codebase right now - regarding the cost model for some transactions. Some built-in - instructions like creating an address lookup table consume more - CUs than the cost model allocates for them, which is only - allowed because the runtime computes requested CUs differently - from the cost model. Rather than implement a broken system, - we'll just permit the risk of slightly overpacking blocks by - ignoring these transactions when it comes to rebating. */ - FD_LOG_INFO(( "Transaction executed %u CUs but only requested %u CUs", executed_cus, requested_cus )); - FD_MCNT_INC( BANK, COST_MODEL_UNDERCOUNT, 1UL ); - txn->bank_cu.rebated_cus = 0U; - continue; - } - txn->bank_cu.rebated_cus = requested_cus - executed_cus; + txn->bank_cu.actual_consumed_cus = non_execution_cus + consumed_cus[ i ]; + FD_TEST( consumed_cus[ i ] <= requested_exec_plus_acct_data_cus ); + txn->bank_cu.rebated_cus = requested_exec_plus_acct_data_cus - consumed_cus[ i ]; } } diff --git a/src/ballet/txn/fd_txn.h b/src/ballet/txn/fd_txn.h index 9eb75c46cf..da5bd73ab8 100644 --- a/src/ballet/txn/fd_txn.h +++ b/src/ballet/txn/fd_txn.h @@ -107,6 +107,12 @@ transaction, using fd_txn_parse() verification rules. */ #define FD_TXN_MIN_SERIALIZED_SZ (134UL) +/* base58 decode of Vote111111111111111111111111111111111111111 */ +static const uchar FD_TXN_VOTE_PROGRAM_ID[FD_TXN_ACCT_ADDR_SZ] = { + 0x07U,0x61U,0x48U,0x1dU,0x35U,0x74U,0x74U,0xbbU,0x7cU,0x4dU,0x76U,0x24U,0xebU,0xd3U,0xbdU,0xb3U, + 0xd8U,0x35U,0x5eU,0x73U,0xd1U,0x10U,0x43U,0xfcU,0x0dU,0xa3U,0x53U,0x80U,0x00U,0x00U,0x00U,0x00U }; + + /* BEGIN Agave limits */ /* "Maximum number of accounts that a transaction may lock. @@ -440,26 +446,22 @@ fd_txn_get_instr_data( fd_txn_instr_t const * instr, return (uchar const *)((ulong)payload + (ulong)instr->data_off); } -/* - * 1. has 1 or 2 signatures - * 2. is legacy message - * 3. has only one instruction - * 4. which must be a Vote instruction +/* A simple vote is a transaction that meets the following criteria: + 1. has 1 or 2 signatures + 2. is legacy transaction + 3. has exactly one instruction + 4. ...which must be a Vote instruction */ static inline int fd_txn_is_simple_vote_transaction( fd_txn_t const * txn, - void const * payload, - uchar const vote_program_id[ static 32 ] ) { + void const * payload ) { fd_acct_addr_t const * addr_base = fd_txn_get_acct_addrs( txn, payload ); - - ulong instr_cnt = txn->instr_cnt; - ulong vote_instr_cnt = 0UL; - for( ulong i=0UL; iinstr_cnt; i++ ) { - ulong prog_id_idx = (ulong)txn->instr[i].program_id; - fd_acct_addr_t const * prog_id = addr_base + prog_id_idx; - vote_instr_cnt += (ulong)(0 == memcmp(prog_id->b, vote_program_id, 32UL) ); - } - return (vote_instr_cnt==1UL) && (instr_cnt==1UL) && (txn->transaction_version==FD_TXN_VLEGACY) && (txn->signature_cnt<3UL); + if( FD_UNLIKELY( txn->instr_cnt!=1UL ) ) return 0; + if( FD_UNLIKELY( txn->transaction_version!=FD_TXN_VLEGACY ) ) return 0; + if( FD_UNLIKELY( txn->signature_cnt>2UL ) ) return 0; + ulong prog_id_idx = (ulong)txn->instr[0].program_id; + fd_acct_addr_t const * prog_id = addr_base + prog_id_idx; + return fd_memeq( prog_id->b, FD_TXN_VOTE_PROGRAM_ID, FD_TXN_ACCT_ADDR_SZ ); } /* fd_txn_align returns the alignment in bytes required of a region of diff --git a/src/disco/pack/fd_compute_budget_program.h b/src/disco/pack/fd_compute_budget_program.h index e52f14354f..11db5dfd00 100644 --- a/src/disco/pack/fd_compute_budget_program.h +++ b/src/disco/pack/fd_compute_budget_program.h @@ -36,13 +36,15 @@ static const uchar FD_COMPUTE_BUDGET_PROGRAM_ID[FD_TXN_ACCT_ADDR_SZ] = { /* Any requests for larger heap frames must be a multiple of 1k or the transaction is malformed. */ -#define FD_COMPUTE_BUDGET_HEAP_FRAME_GRANULARITY (1024UL) +#define FD_COMPUTE_BUDGET_HEAP_FRAME_GRANULARITY ( 1024UL) /* SetComputeUnitPrice specifies the price in "micro-lamports," which is 10^(-6) lamports, so 10^(-15) SOL. */ -#define FD_COMPUTE_BUDGET_MICRO_LAMPORTS_PER_LAMPORT (1000000UL) +#define FD_COMPUTE_BUDGET_MICRO_LAMPORTS_PER_LAMPORT ( 1000000UL) -#define FD_COMPUTE_BUDGET_DEFAULT_INSTR_CU_LIMIT ( 200000UL) -#define FD_COMPUTE_BUDGET_MAX_CU_LIMIT (1400000UL) +#define FD_COMPUTE_BUDGET_DEFAULT_INSTR_CU_LIMIT ( 200000UL) +#define FD_COMPUTE_BUDGET_MAX_CU_LIMIT ( 1400000UL) +#define FD_COMPUTE_BUDGET_HEAP_COST ( 8UL) +#define FD_COMPUTE_BUDGET_ACCOUNT_DATA_COST_PAGE_SIZE (32UL * 1024UL) /* Max loaded data size is 64 MiB */ #define FD_COMPUTE_BUDGET_MAX_LOADED_DATA_SZ (64UL*1024UL*1024UL) @@ -154,7 +156,8 @@ static inline void fd_compute_budget_program_finalize( fd_compute_budget_program_state_t const * state, ulong instr_cnt, ulong * out_rewards, - uint * out_compute ) { + uint * out_compute, + ulong * out_loaded_account_data_cost ) { ulong cu_limit = 0UL; if( FD_LIKELY( (state->flags & FD_COMPUTE_BUDGET_PROGRAM_FLAG_SET_CU)==0U ) ) { /* Use default compute limit */ @@ -165,6 +168,21 @@ fd_compute_budget_program_finalize( fd_compute_budget_program_state_t const * st *out_compute = (uint)cu_limit; + ulong loaded_accounts_data_size = 0UL; + if( FD_LIKELY( (state->flags & FD_COMPUTE_BUDGET_PROGRAM_FLAG_SET_LOADED_DATA_SZ)==0U ) ) { + /* Use default loaded account data size */ + loaded_accounts_data_size = FD_COMPUTE_BUDGET_MAX_LOADED_DATA_SZ; + } else loaded_accounts_data_size = state->loaded_acct_data_sz; + + /* https://github.com/firedancer-io/agave/blob/927c6d30ed5e1baea30c06ebf2aa0c9bae0e1dd1/sdk/fee-structure/src/lib.rs#L122-L129 + + loaded_accounts_data_size <= FD_COMPUTE_BUDGET_MAX_LOADED_DATA_SZ + means no overflow */ + ulong loaded_accounts_data_cost = FD_COMPUTE_BUDGET_HEAP_COST * ( ( + loaded_accounts_data_size + ( FD_COMPUTE_BUDGET_ACCOUNT_DATA_COST_PAGE_SIZE - 1UL ) + ) / FD_COMPUTE_BUDGET_ACCOUNT_DATA_COST_PAGE_SIZE ); + *out_loaded_account_data_cost = loaded_accounts_data_cost; + ulong total_fee = 0UL; /* We need to compute max(ceil((cu_limit * micro_lamports_per_cu)/10^6), diff --git a/src/disco/pack/fd_microblock.h b/src/disco/pack/fd_microblock.h index d47b315a41..8da0d195c9 100644 --- a/src/disco/pack/fd_microblock.h +++ b/src/disco/pack/fd_microblock.h @@ -54,11 +54,11 @@ struct __attribute__((aligned(64))) fd_txn_p { union { struct { uint non_execution_cus; - uint requested_execution_cus; + uint requested_exec_plus_acct_data_cus; } pack_cu; /* Populated by pack. Bank reads these to populate the other struct of the union. */ struct { - uint rebated_cus; /* requested_execution_cus-real execution CUs. Pack reads this for CU rebating. */ - uint actual_consumed_cus; /* non_execution_cus+real execution CUs. PoH reads this for block CU counting. */ + uint rebated_cus; /* requested_exec_plus_acct_data_cus-actual used CUs. Pack reads this for CU rebating. */ + uint actual_consumed_cus; /* non_execution_cus+real execution CUs+real account data cus. PoH reads this for block CU counting. */ } bank_cu; /* Populated by bank. */ ulong blockhash_slot; /* Slot provided by resolv tile when txn arrives at the pack tile. Used when txn is in extra storage in pack. */ }; diff --git a/src/disco/pack/fd_pack.c b/src/disco/pack/fd_pack.c index c712063a95..a80a8c0042 100644 --- a/src/disco/pack/fd_pack.c +++ b/src/disco/pack/fd_pack.c @@ -871,12 +871,13 @@ fd_pack_estimate_rewards_and_compute( fd_txn_e_t * txne, fd_txn_t * txn = TXN(txne->txnp); ulong sig_rewards = FD_PACK_FEE_PER_SIGNATURE * txn->signature_cnt; /* Easily in [5000, 635000] */ - ulong execution_cus; - ulong adtl_rewards; + ulong requested_execution_cus; + ulong priority_rewards; ulong precompile_sigs; - ulong cost = fd_pack_compute_cost( txn, txne->txnp->payload, &txne->txnp->flags, &execution_cus, &adtl_rewards, &precompile_sigs ); + ulong requested_loaded_accounts_data_cost; + ulong cost_estimate = fd_pack_compute_cost( txn, txne->txnp->payload, &txne->txnp->flags, &requested_execution_cus, &priority_rewards, &precompile_sigs, &requested_loaded_accounts_data_cost ); - if( FD_UNLIKELY( !cost ) ) return 0; + if( FD_UNLIKELY( !cost_estimate ) ) return 0; /* precompile_sigs <= 16320, so after the addition, sig_rewards < 83,000,000 */ @@ -888,10 +889,10 @@ fd_pack_estimate_rewards_and_compute( fd_txn_e_t * txne, fd_acct_addr_t const * acct_addr = fd_txn_get_acct_addrs( txn, txnp->payload ) + (ulong)prog_id_idx; } */ - out->rewards = (adtl_rewards < (UINT_MAX - sig_rewards)) ? (uint)(sig_rewards + adtl_rewards) : UINT_MAX; - out->compute_est = (uint)cost; - out->txn->pack_cu.requested_execution_cus = (uint)execution_cus; - out->txn->pack_cu.non_execution_cus = (uint)(cost - execution_cus); + out->rewards = (priority_rewards < (UINT_MAX - sig_rewards)) ? (uint)(sig_rewards + priority_rewards) : UINT_MAX; + out->compute_est = (uint)cost_estimate; + out->txn->pack_cu.requested_exec_plus_acct_data_cus = (uint)(requested_execution_cus + requested_loaded_accounts_data_cost); + out->txn->pack_cu.non_execution_cus = (uint)(cost_estimate - requested_execution_cus - requested_loaded_accounts_data_cost); return fd_int_if( txne->txnp->flags & FD_TXN_P_FLAGS_IS_SIMPLE_VOTE, 1, 2 ); } @@ -1834,7 +1835,7 @@ fd_pack_schedule_impl( fd_pack_t * pack, fd_memcpy( out->payload, cur->txn->payload, cur->txn->payload_sz ); fd_memcpy( TXN(out), txn, fd_txn_footprint( txn->instr_cnt, txn->addr_table_lookup_cnt ) ); out->payload_sz = cur->txn->payload_sz; - out->pack_cu.requested_execution_cus = cur->txn->pack_cu.requested_execution_cus; + out->pack_cu.requested_exec_plus_acct_data_cus = cur->txn->pack_cu.requested_exec_plus_acct_data_cus; out->pack_cu.non_execution_cus = cur->txn->pack_cu.non_execution_cus; out->flags = cur->txn->flags; } @@ -2268,7 +2269,7 @@ fd_pack_try_schedule_bundle( fd_pack_t * pack, fd_memcpy( out->payload, cur->txn->payload, cur->txn->payload_sz ); fd_memcpy( TXN(out), txn, fd_txn_footprint( txn->instr_cnt, txn->addr_table_lookup_cnt ) ); out->payload_sz = cur->txn->payload_sz; - out->pack_cu.requested_execution_cus = cur->txn->pack_cu.requested_execution_cus; + out->pack_cu.requested_exec_plus_acct_data_cus = cur->txn->pack_cu.requested_exec_plus_acct_data_cus; out->pack_cu.non_execution_cus = cur->txn->pack_cu.non_execution_cus; out->flags = cur->txn->flags; out++; @@ -2358,7 +2359,7 @@ fd_pack_schedule_next_microblock( fd_pack_t * pack, total_cus = fd_ulong_min( total_cus, pack->lim->max_cost_per_block - pack->cumulative_block_cost ); ulong vote_cus = fd_ulong_min( (ulong)((float)total_cus * vote_fraction), pack->lim->max_vote_cost_per_block - pack->cumulative_vote_cost ); - ulong vote_reserved_txns = fd_ulong_min( vote_cus/FD_PACK_TYPICAL_VOTE_COST, + ulong vote_reserved_txns = fd_ulong_min( vote_cus/FD_PACK_SIMPLE_VOTE_COST, (ulong)((float)pack->lim->max_txn_per_microblock * vote_fraction) ); diff --git a/src/disco/pack/fd_pack.h b/src/disco/pack/fd_pack.h index 32b3d7dc00..48e2ffcca5 100644 --- a/src/disco/pack/fd_pack.h +++ b/src/disco/pack/fd_pack.h @@ -518,14 +518,15 @@ void fd_pack_set_initializer_bundles_ready( fd_pack_t * pack ); though the block-level limits are respected. Both cases: - The non_execution_cus and requested_execution_cus fields of each - transaction will be populated with the non execution CUs and - requested execution CUs, respectively. The sum of these two values - is the total cost of the transaction, i.e. what is used for all - limits, including the total_cus value. The lower 3 bits of the flags - field will be populated (simple vote, bundle, initializer bundle). - Inspecting these flags is the proper way to tell which codepath - executed. + The non_execution_cus and requested_exec_plus_acct_data_cus fields of + each transaction will be populated with the non execution CUs and + requested execution CUs (including cus derived from the requested + loaded accounts data size), respectively. The sum of these two + values is the total cost of the transaction, i.e. what is used for + all limits, including the total_cus value. The lower 3 bits of the + flags field will be populated (simple vote, bundle, initializer + bundle). Inspecting these flags is the proper way to tell which + codepath executed. Returns the number of transactions in the scheduled microblock or bundle. The return value may be 0 if there are no eligible diff --git a/src/disco/pack/fd_pack_cost.h b/src/disco/pack/fd_pack_cost.h index 79c356cb7a..aab4d43c05 100644 --- a/src/disco/pack/fd_pack_cost.h +++ b/src/disco/pack/fd_pack_cost.h @@ -3,19 +3,32 @@ #include "../../ballet/fd_ballet_base.h" #include "fd_compute_budget_program.h" #include "../../flamenco/runtime/fd_system_ids_pp.h" +#include "../../ballet/txn/fd_txn.h" /* The functions in this header implement the transaction cost model that is soon to be part of consensus. The cost model consists of several components: - * per-signature cost - * per-write-lock cost - * instruction data length cost - * built-in execution cost - * BPF execution cost - These are all summed to determine the total cost. Additionally, this - header provides a method for determining if a transaction is a simple - vote transaction, in which case its costs are used slightly - differently. */ + * per-signature cost: The costs associated with transaction + signatures + signatures from precompiles (ED25519 + SECP256K1) + * per-write-lock cost: cost assiciated with aquiring write locks + for writable accounts listed in the transaction. + * instruction data length cost: The fixed cost for each instruction + data byte in the transaction payload. + * built-in execution cost: The fixed cost associated with "builtin" + instructions. "What are builtins" is defined here: + https://github.com/anza-xyz/agave/blob/1baa4033e0d2d4175373f07b73ddda2f3cc0a8d6/builtins-default-costs/src/lib.rs#L120-L200 + After SIMD 170, all builtins have a fixed cost of 3000 cus. + * BPF execution cost: The costs assosiated with any instruction + that is not a builtin. This value comes from the VM after + transaction execution. + * loaded accounts data cost: Costs associated with all the account + data loaded from the chain. This value is known in the + transaction loading stage, after accounts data is loaded. + These are all summed to determine the total transaction cost. */ + +/* The exception to these costs are simple vote transactions, incur a + fixed cost regardless of their actual execution cost or account data + loaded. */ /* To compute the built-in cost, we need to check a table. The table is known ahead of time though, so we can build a perfect hash @@ -51,9 +64,7 @@ typedef struct fd_pack_builtin_prog_cost fd_pack_builtin_prog_cost_t; /* The cost model estimates 200,000 CUs for builtin programs that were migrated to BPF */ -#define VOTE_PROG_COST 3000UL - -#define MAP_PERFECT_0 ( VOTE_PROG_ID ), .cost_per_instr= VOTE_PROG_COST +#define MAP_PERFECT_0 ( VOTE_PROG_ID ), .cost_per_instr= 3000UL #define MAP_PERFECT_1 ( SYS_PROG_ID ), .cost_per_instr= 3000UL #define MAP_PERFECT_2 ( COMPUTE_BUDGET_PROG_ID ), .cost_per_instr= 3000UL #define MAP_PERFECT_3 ( BPF_UPGRADEABLE_PROG_ID ), .cost_per_instr= 3000UL @@ -70,9 +81,11 @@ typedef struct fd_pack_builtin_prog_cost fd_pack_builtin_prog_cost_t; a16,a17,a18,a19,a20,a21,a22,a23,a24,a25,a26,a27,a28,a29,a30,a31) \ PERFECT_HASH( ((uint)a08 | ((uint)a09<<8) | ((uint)a10<<16) | ((uint)a11<<24)) ) -#define FD_PACK_COST_PER_SIGNATURE (720UL) -#define FD_PACK_COST_PER_WRITABLE_ACCT (300UL) -#define FD_PACK_INV_COST_PER_INSTR_DATA_BYTE ( 4UL) +#define FD_PACK_COST_PER_SIGNATURE ( 720UL) +#define FD_PACK_COST_PER_ED25519_SIGNATURE ( 2400UL) +#define FD_PACK_COST_PER_SECP256K1_SIGNATURE ( 6690UL) +#define FD_PACK_COST_PER_WRITABLE_ACCT ( 300UL) +#define FD_PACK_INV_COST_PER_INSTR_DATA_BYTE ( 4UL) /* The computation here is similar to the computation for the max fd_txn_t size. There are various things a transaction can include @@ -103,6 +116,9 @@ typedef struct fd_pack_builtin_prog_cost fd_pack_builtin_prog_cost_t; Finally, with any bytes that remain, we can add them to one of the instruction datas for 0.25 CUs/byte. + Note that by default, 64MiB of data are assumed for the loaded + accounts data size cost. This corresponds (currently) to 16384 CUs. + This gives a transaction that looks like Field bytes consumed CUs used sig cnt 1 0 @@ -119,29 +135,35 @@ typedef struct fd_pack_builtin_prog_cost fd_pack_builtin_prog_cost_t; Compute budget program ix 8 151.25 62 dummy BPF upg ixs 186 146,940 1 dummy non-builtin ix 8 1,400,001.25 + loaded accts data cost 0 16384 + --------------------------------------------------------------- - 1,232 1,556,782 + 1,232 1,573,166 One of the main take-aways from this is that the cost of a transaction easily fits in a uint. */ -#define FD_PACK_MAX_TXN_COST (1556782UL) +#define FD_PACK_MAX_TXN_COST (1573166UL) FD_STATIC_ASSERT( FD_PACK_MAX_TXN_COST < (ulong)UINT_MAX, fd_pack_max_cost ); /* Every transaction has at least a fee payer, a writable signer. */ #define FD_PACK_MIN_TXN_COST (FD_PACK_COST_PER_SIGNATURE+FD_PACK_COST_PER_WRITABLE_ACCT) -/* A typical vote transaction has the authorized voter (writable - signer), the vote account (writable non-signer), and the vote program - (readonly). Then it has one instruction, a built-in to the vote - program, which is typically 116 bytes long, but occasionally a little - less than that. The mean over several million slots of vote - transactions (10B votes) is 115.990 bytes. */ -static const ulong FD_PACK_TYPICAL_VOTE_COST = ( FD_PACK_COST_PER_SIGNATURE + - 2UL*FD_PACK_COST_PER_WRITABLE_ACCT + - 116UL/FD_PACK_INV_COST_PER_INSTR_DATA_BYTE + - VOTE_PROG_COST ); -#undef VOTE_PROG_COST +/* https://github.com/anza-xyz/agave/blob/v2.0.1/programs/vote/src/vote_processor.rs#L55 */ + +#define FD_PACK_VOTE_DEFAULT_COMPUTE_UNITS 2100UL + +/* A simple vote transaction has the authorized voter (writable + signer), the vote account (writable non-signer), clock sysvar, slot + hashes sysvar (both readonly), and the vote program (readonly). Then + it has one instruction a built-in to the vote program, which has a + fixed cost of DEFAULT_COMPUTE_UNITS, and an instruction data cost of + 8. + + See https://github.com/firedancer-io/agave/blob/v2.1.11-fd/sdk/src/simple_vote_transaction_checker.rs */ +static const ulong FD_PACK_SIMPLE_VOTE_COST = ( FD_PACK_COST_PER_SIGNATURE + + 2UL*FD_PACK_COST_PER_WRITABLE_ACCT + + FD_PACK_VOTE_DEFAULT_COMPUTE_UNITS + + 8 ); /* Computes the total cost and a few related properties for the @@ -159,9 +181,12 @@ static const ulong FD_PACK_TYPICAL_VOTE_COST = ( FD_PACK_COST_PER_SIGNATURE per-signature fee). This value is in [0, ULONG_MAX]. If opt_precompile_sig_cnt is non-null, on success it will contain the total number of signatures in precompile instructions, namely Keccak - and Ed25519 signature verification programs. This value is in [0, + and Ed25519 signature verification programs. This value is in [0, 256*64]. Note that this does not do full parsing of the precompile - instruction, and it may be malformed. + instruction, and it may be malformed. If + opt_loaded_accounts_data_cost is non-null, on success it will contain + the total requested cost due to loaded accounts data. This value is + in [0, the returned value). On failure, returns 0 and does not modify the value pointed to by flags, opt_execution_cost, opt_fee, or opt_precompile_sig_cnt. */ @@ -171,35 +196,47 @@ fd_pack_compute_cost( fd_txn_t const * txn, uint * flags, ulong * opt_execution_cost, ulong * opt_fee, - ulong * opt_precompile_sig_cnt ) { + ulong * opt_precompile_sig_cnt, + ulong * opt_loaded_accounts_data_cost ) { #define ROW(x) fd_pack_builtin_tbl + MAP_PERFECT_HASH_PP( x ) - fd_pack_builtin_prog_cost_t const * compute_budget_row = ROW( COMPUTE_BUDGET_PROG_ID ); - fd_pack_builtin_prog_cost_t const * vote_row = ROW( VOTE_PROG_ID ); fd_pack_builtin_prog_cost_t const * ed25519_precompile_row = ROW( ED25519_SV_PROG_ID ); - fd_pack_builtin_prog_cost_t const * keccak_precompile_row = ROW( KECCAK_SECP_PROG_ID ); fd_pack_builtin_prog_cost_t const * secp256r1_precomp_row = ROW( SECP256R1_PROG_ID ); #undef ROW - /* We need to be mindful of overflow here, but it's not terrible. - signature_cost <= FD_TXN_ACCT_ADDR_MAX*720, - writable_cost <= FD_TXN_ACCT_ADDR_MAX*300 */ + /* special handling for simple votes */ + if( FD_UNLIKELY( fd_txn_is_simple_vote_transaction( txn, payload ) ) ) { +#if DETAILED_LOGGING + FD_BASE58_ENCODE_64_BYTES( (const uchar *)fd_txn_get_signatures(txn, payload), signature_cstr ); + FD_LOG_NOTICE(( "TXN SIMPLE_VOTE signature[%s] total_cost[%lu]", signature_cstr, FD_PACK_SIMPLE_VOTE_COST)); +#endif + *flags |= FD_TXN_P_FLAGS_IS_SIMPLE_VOTE; + fd_ulong_store_if( !!opt_execution_cost, opt_execution_cost, FD_PACK_VOTE_DEFAULT_COMPUTE_UNITS ); + fd_ulong_store_if( !!opt_fee, opt_fee, 0 ); + fd_ulong_store_if( !!opt_precompile_sig_cnt, opt_precompile_sig_cnt, 0 ); + fd_ulong_store_if( !!opt_loaded_accounts_data_cost, opt_loaded_accounts_data_cost, 0 ); + return FD_PACK_SIMPLE_VOTE_COST; + } + + *flags &= ~FD_TXN_P_FLAGS_IS_SIMPLE_VOTE; - ulong signature_cost = FD_PACK_COST_PER_SIGNATURE * fd_txn_account_cnt( txn, FD_TXN_ACCT_CAT_SIGNER ); + /* We need to be mindful of overflow here, but it's not terrible. + signature_cost < FD_TXN_ACCT_ADDR_MAX*720 + FD_TXN_INSTR_MAX * UCHAR_MAX * 6690, + writable_cost <= FD_TXN_ACCT_ADDR_MAX*300 */ + ulong signature_cnt = fd_txn_account_cnt( txn, FD_TXN_ACCT_CAT_SIGNER ); + ulong signature_cost = FD_PACK_COST_PER_SIGNATURE * signature_cnt; ulong writable_cost = FD_PACK_COST_PER_WRITABLE_ACCT * fd_txn_account_cnt( txn, FD_TXN_ACCT_CAT_WRITABLE ); ulong instr_data_sz = 0UL; /* < FD_TPU_MTU */ ulong builtin_cost = 0UL; /* <= 2370*FD_TXN_INSTR_MAX */ ulong non_builtin_cnt = 0UL; /* <= FD_TXN_INSTR_MAX */ - ulong vote_instr_cnt = 0UL; /* <= FD_TXN_INSTR_MAX */ ulong precompile_sig_cnt = 0UL; /* <= FD_TXN_INSTR_MAX * UCHAR_MAX */ fd_acct_addr_t const * addr_base = fd_txn_get_acct_addrs( txn, payload ); fd_compute_budget_program_state_t cbp[1]; fd_compute_budget_program_init( cbp ); - for( ulong i=0UL; iinstr_cnt; i++ ) { instr_data_sz += txn->instr[i].data_sz; @@ -216,21 +253,25 @@ fd_pack_compute_cost( fd_txn_t const * txn, if( FD_UNLIKELY( in_tbl==compute_budget_row ) ) { if( FD_UNLIKELY( 0==fd_compute_budget_program_parse( payload+txn->instr[i].data_off, txn->instr[i].data_sz, cbp ) ) ) return 0UL; - } else if( FD_UNLIKELY( (in_tbl==ed25519_precompile_row) | (in_tbl==keccak_precompile_row) | (in_tbl==secp256r1_precomp_row) ) ) { + } else if( FD_UNLIKELY( (in_tbl==ed25519_precompile_row) ) ) { /* First byte is # of signatures. Branchless tail reading here is probably okay, but this seems safer. */ - precompile_sig_cnt += (txn->instr[i].data_sz>0) ? (ulong)payload[ txn->instr[i].data_off ] : 0UL; + ulong ed25519_signature_count = (txn->instr[i].data_sz>0) ? (ulong)payload[ txn->instr[i].data_off ] : 0UL; + precompile_sig_cnt += ed25519_signature_count; + signature_cost += ed25519_signature_count * FD_PACK_COST_PER_ED25519_SIGNATURE; + } else if( FD_UNLIKELY( (in_tbl==secp256r1_precomp_row) ) ) { + ulong secp256r1_signature_count = (txn->instr[i].data_sz>0) ? (ulong)payload[ txn->instr[i].data_off ] : 0UL; + precompile_sig_cnt += secp256r1_signature_count; + signature_cost += secp256r1_signature_count * FD_PACK_COST_PER_SECP256K1_SIGNATURE; } - - vote_instr_cnt += (ulong)(in_tbl==vote_row); - } ulong instr_data_cost = instr_data_sz / FD_PACK_INV_COST_PER_INSTR_DATA_BYTE; /* <= 320 */ ulong fee[1]; uint compute[1]; - fd_compute_budget_program_finalize( cbp, txn->instr_cnt, fee, compute ); + ulong loaded_account_data_cost[1]; + fd_compute_budget_program_finalize( cbp, txn->instr_cnt, fee, compute, loaded_account_data_cost ); non_builtin_cnt = fd_ulong_min( non_builtin_cnt, FD_COMPUTE_BUDGET_MAX_CU_LIMIT/FD_COMPUTE_BUDGET_DEFAULT_INSTR_CU_LIMIT ); @@ -239,16 +280,19 @@ fd_pack_compute_cost( fd_txn_t const * txn, non_builtin_cnt*FD_COMPUTE_BUDGET_DEFAULT_INSTR_CU_LIMIT ); /* <= FD_COMPUTE_BUDGET_MAX_CU_LIMIT */ + fd_ulong_store_if( !!opt_execution_cost, opt_execution_cost, builtin_cost + non_builtin_cost ); + fd_ulong_store_if( !!opt_fee, opt_fee, *fee ); + fd_ulong_store_if( !!opt_precompile_sig_cnt, opt_precompile_sig_cnt, precompile_sig_cnt ); + fd_ulong_store_if( !!opt_loaded_accounts_data_cost, opt_loaded_accounts_data_cost, *loaded_account_data_cost ); - if( FD_LIKELY( (vote_instr_cnt==1UL) & (txn->instr_cnt==1UL) ) ) *flags |= FD_TXN_P_FLAGS_IS_SIMPLE_VOTE; - else *flags &= ~FD_TXN_P_FLAGS_IS_SIMPLE_VOTE; - - fd_ulong_store_if( !!opt_execution_cost, opt_execution_cost, builtin_cost + non_builtin_cost ); - fd_ulong_store_if( !!opt_fee, opt_fee, *fee ); - fd_ulong_store_if( !!opt_precompile_sig_cnt, opt_precompile_sig_cnt, precompile_sig_cnt ); +#if DETAILED_LOGGING + FD_BASE58_ENCODE_64_BYTES( (const uchar *)fd_txn_get_signatures(txn, payload), signature_cstr ); + FD_LOG_NOTICE(( "TXN signature[%s] signature_cost[%lu] writable_cost[%lu] builtin_cost[%lu] instr_data_cost[%lu] non_builtin_cost[%lu] loaded_account_data_cost[%lu] precompile_sig_cnt[%lu] fee[%lu]", + signature_cstr, signature_cost, writable_cost, builtin_cost, instr_data_cost, non_builtin_cost, *loaded_account_data_cost, precompile_sig_cnt, *fee)); +#endif /* <= FD_PACK_MAX_COST, so no overflow concerns */ - return signature_cost + writable_cost + builtin_cost + instr_data_cost + non_builtin_cost; + return signature_cost + writable_cost + builtin_cost + instr_data_cost + non_builtin_cost + *loaded_account_data_cost; } #undef MAP_PERFECT_HASH_PP #undef PERFECT_HASH diff --git a/src/disco/pack/test_compute_budget_program.c b/src/disco/pack/test_compute_budget_program.c index 2bfc8022af..f770139751 100644 --- a/src/disco/pack/test_compute_budget_program.c +++ b/src/disco/pack/test_compute_budget_program.c @@ -5,17 +5,17 @@ FD_IMPORT_BINARY( txn2, "src/disco/pack/fixtures/txn2.bin" ); /* 500k CU, 15001 FD_IMPORT_BINARY( txn3, "src/disco/pack/fixtures/txn3.bin" ); /* Just 1M CU, no extra fee */ FD_IMPORT_BINARY( txn4, "src/disco/pack/fixtures/txn4.bin" ); /* 75k CU, 20001 ulamports per CU, the CU request has trailing data */ FD_IMPORT_BINARY( txn5, "src/disco/pack/fixtures/txn5.bin" ); /* Requests 6M CUs, so only allotted 1.4M CUs, total fee of 33,000 lamports */ -FD_IMPORT_BINARY( txn6, "src/disco/pack/fixtures/txn6.bin" ); /* Includes a requested_loaded_accounts_data_size_limit instruction */ +FD_IMPORT_BINARY( txn6, "src/disco/pack/fixtures/txn6.bin" ); /* Includes a requested_loaded_accounts_data_size_limit instruction (1000000 bytes) */ FD_IMPORT_BINARY( txn7, "src/disco/pack/fixtures/txn7.bin" ); /* Requests additional heap */ - uchar parsed[FD_TXN_MAX_SZ]; void test_txn( uchar const * payload, ulong payload_sz, ulong expected_max_cu, - ulong expected_fee_lamports ) { /* Excludes per-signature fee */ + ulong expected_fee_lamports, + ulong expected_loaded_accounts_data_cost ) { /* Excludes per-signature fee */ FD_TEST( fd_txn_parse( payload, payload_sz, parsed, NULL ) ); fd_txn_t * txn = (fd_txn_t*)parsed; fd_compute_budget_program_state_t state; @@ -28,9 +28,11 @@ test_txn( uchar const * payload, } ulong rewards = 0UL; uint compute = 0U; - fd_compute_budget_program_finalize( &state, txn->instr_cnt, &rewards, &compute ); - FD_TEST( rewards==expected_fee_lamports ); - FD_TEST( (ulong)compute==expected_max_cu ); + ulong loaded_accounts_data_cost = 0UL; + fd_compute_budget_program_finalize( &state, txn->instr_cnt, &rewards, &compute, &loaded_accounts_data_cost); + FD_TEST( rewards == expected_fee_lamports ); + FD_TEST( (ulong)compute == expected_max_cu ); + FD_TEST( loaded_accounts_data_cost == expected_loaded_accounts_data_cost ); } FD_FN_CONST int @@ -68,24 +70,24 @@ main( int argc, fd_rng_t _rng[1]; fd_rng_t * rng = fd_rng_join( fd_rng_new( _rng, 0U, 0UL ) ); - test_txn( txn1, txn1_sz, 1400000UL, 280000UL ); - test_txn( txn2, txn2_sz, 500000UL, 7501UL ); - test_txn( txn3, txn3_sz, 1000000UL, 0UL ); - test_txn( txn4, txn4_sz, 75000UL, 1501UL ); - test_txn( txn5, txn5_sz, 1400000UL, 28000UL ); - test_txn( txn6, txn6_sz, 60000UL, 5400UL ); - test_txn( txn7, txn7_sz, 1400000UL, 0UL ); + test_txn( txn1, txn1_sz, 1400000UL, 280000UL, 16384UL ); + test_txn( txn2, txn2_sz, 500000UL, 7501ULL, 16384UL ); + test_txn( txn3, txn3_sz, 1000000UL, 0ULL, 16384UL ); + test_txn( txn4, txn4_sz, 75000UL, 1501ULL, 16384UL ); + test_txn( txn5, txn5_sz, 1400000UL, 28000ULL, 16384UL ); + test_txn( txn6, txn6_sz, 60000UL, 5400ULL, 248UL ); + test_txn( txn7, txn7_sz, 1400000UL, 0ULL, 16384UL ); uchar _txn2[ txn2_sz ]; fd_memcpy( _txn2, txn2, txn2_sz ); uint * cu_limit = (uint *) &_txn2[ 260 ]; ulong * ulamports = (ulong *) &_txn2[ 268 ]; - *cu_limit = 1000000U; *ulamports = 1000000UL; test_txn( _txn2, txn2_sz, 1000000UL, 1000000UL ); /* No overflow */ - *cu_limit = 1000000U; *ulamports = ULONG_MAX>>1; test_txn( _txn2, txn2_sz, 1000000UL, ULONG_MAX>>1 ); /* Product>2^64 */ - *cu_limit = 1400000U; *ulamports = ULONG_MAX; test_txn( _txn2, txn2_sz, 1400000UL, ULONG_MAX ); /* Result>2^64 */ - *cu_limit = 1400000U; *ulamports = 1UL<<44; test_txn( _txn2, txn2_sz, 1400000UL, 24629060462183UL ); /* Product<2^64 */ - *cu_limit = 1U; *ulamports = 1UL; test_txn( _txn2, txn2_sz, 1UL, 1UL ); /* Test ceil */ + *cu_limit = 1000000U; *ulamports = 1000000UL; test_txn( _txn2, txn2_sz, 1000000UL, 1000000UL, 16384UL ); /* No overflow */ + *cu_limit = 1000000U; *ulamports = ULONG_MAX>>1; test_txn( _txn2, txn2_sz, 1000000UL, ULONG_MAX>>1, 16384UL ); /* Product>2^64 */ + *cu_limit = 1400000U; *ulamports = ULONG_MAX; test_txn( _txn2, txn2_sz, 1400000UL, ULONG_MAX, 16384UL ); /* Result>2^64 */ + *cu_limit = 1400000U; *ulamports = 1UL<<44; test_txn( _txn2, txn2_sz, 1400000UL, 24629060462183UL, 16384UL ); /* Product<2^64 */ + *cu_limit = 1U; *ulamports = 1UL; test_txn( _txn2, txn2_sz, 1UL, 1UL, 16384UL ); /* Test ceil */ FD_TEST( test_duplicate( 1, 1, 0, 0, 0 ) == 0 ); FD_TEST( test_duplicate( 2, 0, 0, 0, 0 ) == 0 ); diff --git a/src/disco/pack/test_pack.c b/src/disco/pack/test_pack.c index 0ebd503e70..889da92adb 100644 --- a/src/disco/pack/test_pack.c +++ b/src/disco/pack/test_pack.c @@ -1,5 +1,6 @@ #include "../../ballet/fd_ballet.h" #include "fd_pack.h" +#include "fd_pack_cost.h" #include "fd_compute_budget_program.h" #include "../../ballet/txn/fd_txn.h" #include "../../ballet/base58/fd_base58.h" @@ -7,7 +8,6 @@ #include FD_IMPORT_BINARY( sample_vote, "src/disco/pack/sample_vote.bin" ); -#define SAMPLE_VOTE_COST (4335UL) #define MAX_TEST_TXNS (1024UL) #define MAX_DATA_PER_BLOCK (5UL*1024UL*1024UL) @@ -80,18 +80,24 @@ init_all( ulong pack_depth, each character in reads and writes one account for each character in writes. The characters before the nul-terminator in reads and writes should be in [0x30, 0x70), basically numbers and uppercase letters. - Adds a unique signer. Packing should estimate compute usage near the - specified value. Fee will be set to 5^priority, so that even with a - large stall, it should still schedule in decreasing priority order. - priority should be in (0, 13.5]. Stores the created transaction in - txn_scratch[ i ] and payload_scratch[ i ]. Returns the priority fee - in lamports. */ -static ulong + Adds a unique signer. A computeBudgetInstruction will be included + with compute requested cus and another instruction will be added + requesting loaded_data_sz bytes of accounts data. Fee will be set to + 5^priority, so that even with a large stall, it should still schedule + in decreasing priority order. priority should be in (0, 13.5]. + Stores the created transaction in txn_scratch[ i ] and + payload_scratch[ i ]. If priority_fees is non-null, it will contain + the priority fee in lamports. If pack_cost_estimate is non-null, it + will contain the cost estimate used by pack when packing blocks. */ +static void make_transaction( ulong i, uint compute, + uint loaded_data_sz, double priority, char const * writes, - char const * reads ) { + char const * reads, + ulong * priority_fees, + ulong * pack_cost_estimate ) { uchar * p = payload_scratch[ i ]; uchar * p_base = p; fd_txn_t * t = (fd_txn_t*) txn_scratch[ i ]; @@ -113,7 +119,7 @@ make_transaction( ulong i, t->acct_addr_off = FD_TXN_SIGNATURE_SZ+1UL; /* Add the signer */ - *p = 's' + 0x80; fd_memcpy( p+1, &i, sizeof(ulong) ); memset( p+9, 'S', 32-9 ); p += FD_TXN_ACCT_ADDR_SZ; + *p = 's' + 0x80; fd_memcpy( p+1, &i, sizeof(ulong) ); memset( p+9, 'S', FD_TXN_ACCT_ADDR_SZ-9 ); p += FD_TXN_ACCT_ADDR_SZ; /* Add the writable accounts */ for( ulong i = 0UL; writes[i] != '\0'; i++ ) { memset( p, writes[i], FD_TXN_ACCT_ADDR_SZ ); @@ -133,7 +139,7 @@ make_transaction( ulong i, t->addr_table_lookup_cnt = 0; t->addr_table_adtl_writable_cnt = 0; t->addr_table_adtl_cnt = 0; - t->instr_cnt = (ushort)(2UL + (ulong)fd_uint_popcnt( compute )); + t->instr_cnt = (ushort)(3UL + (ulong)fd_uint_popcnt( compute )); uchar prog_start = (uchar)(1UL+strlen( writes )); @@ -153,11 +159,22 @@ make_transaction( ulong i, t->instr[ 1 ].acct_off = (ushort)(p - p_base); t->instr[ 1 ].data_off = (ushort)(p - p_base); + /* 3 corresponds to SetComputeUnitPrice */ ulong rewards_per_cu = (ulong) (pow( 5.0, priority )*10000.0 / (double)compute); *p = 3; fd_memcpy( p+1, &rewards_per_cu, sizeof(ulong) ); p += 9UL; - ulong j = 2UL; + t->instr[ 2 ].program_id = prog_start; + t->instr[ 2 ].acct_cnt = 0; + t->instr[ 2 ].data_sz = 5; + t->instr[ 2 ].acct_off = (ushort)(p - p_base); + t->instr[ 2 ].data_off = (ushort)(p - p_base); + + /* 4 corresponds to SetLoadedAccountsDataSizeLimit */ + *p = 4; fd_memcpy( p+1, &loaded_data_sz, sizeof(uint) ); + p += 5UL; + + ulong j = 3UL; for( uint i = 0U; i<32U; i++ ) { if( compute & (1U << i) ) { *p = (uchar)i; @@ -172,7 +189,9 @@ make_transaction( ulong i, } payload_sz[ i ] = (ulong)(p-p_base); - return (rewards_per_cu * compute + 999999UL)/1000000UL; + uint flags; + fd_ulong_store_if( !!priority_fees, priority_fees, (rewards_per_cu * compute + 999999UL)/1000000UL ); + fd_ulong_store_if( !!pack_cost_estimate, pack_cost_estimate, fd_pack_compute_cost((fd_txn_t const *)txn_scratch[ i ], payload_scratch[ i ], &flags, NULL, NULL, NULL, NULL) ); } static void @@ -237,12 +256,13 @@ schedule_validate_microblock( fd_pack_t * pack, ulong rewards = 0UL; uint compute = 0U; + ulong requested_loaded_accounts_data_cost = 0UL; if( FD_LIKELY( txn->instr_cnt>2UL ) ) { fd_txn_instr_t ix = txn->instr[0]; /* For these transactions, the compute budget instr is always the first 2*/ FD_TEST( fd_compute_budget_program_parse( txnp->payload + ix.data_off, ix.data_sz, &cbp ) ); ix = txn->instr[1]; FD_TEST( fd_compute_budget_program_parse( txnp->payload + ix.data_off, ix.data_sz, &cbp ) ); - fd_compute_budget_program_finalize( &cbp, txn->instr_cnt, &rewards, &compute ); + fd_compute_budget_program_finalize( &cbp, txn->instr_cnt, &rewards, &compute, &requested_loaded_accounts_data_cost ); } /* else it's a vote */ total_rewards += rewards; @@ -290,16 +310,19 @@ void test0( void ) { FD_LOG_NOTICE(( "TEST 0" )); fd_pack_t * pack = init_all( 128UL, 3UL, 128UL, &outcome ); ulong i = 0UL; - ulong rewards = 0UL; - rewards += make_transaction( i, 500U, 11.0, "A", "B" ); insert( i++, pack ); - rewards += make_transaction( i, 500U, 10.0, "C", "D" ); insert( i++, pack ); - rewards += make_transaction( i, 800U, 10.0, "EFGH", "D" ); insert( i++, pack ); - schedule_validate_microblock( pack, 30000UL, 0.0f, 3UL, rewards, 0UL, &outcome ); - - make_transaction( i, 500U, 10.0, "D", "I" ); insert( i++, pack ); - schedule_validate_microblock( pack, 30000UL, 0.0f, 0UL, 0UL, 1UL, &outcome ); /* Can't schedule because conflict*/ - schedule_validate_microblock( pack, 30000UL, 0.0f, 0UL, 0UL, 2UL, &outcome ); /* conflict continues ... */ - schedule_validate_microblock( pack, 30000UL, 0.0f, 1UL, 0UL, 0UL, &outcome ); /* conflict gone.*/ + ulong reward; + ulong cost_estimate; + ulong total_rewards = 0UL; + ulong total_cost_estimate = 0UL; + make_transaction( i, 500U, 500U, 11.0, "A", "B", &reward, &cost_estimate ); insert( i++, pack ); total_cost_estimate += cost_estimate; total_rewards += reward; + make_transaction( i, 500U, 500U, 10.0, "C", "D", &reward, &cost_estimate ); insert( i++, pack ); total_cost_estimate += cost_estimate; total_rewards += reward; + make_transaction( i, 800U, 500U, 10.0, "EFGH", "D", &reward, &cost_estimate ); insert( i++, pack ); total_cost_estimate += cost_estimate; total_rewards += reward; + schedule_validate_microblock( pack, total_cost_estimate, 0.0f, 3UL, total_rewards, 0UL, &outcome ); + + make_transaction( i, 500U, 500U, 10.0, "D", "I", &reward, &cost_estimate ); insert( i++, pack ); total_cost_estimate += cost_estimate; total_rewards += reward; + schedule_validate_microblock( pack, total_cost_estimate, 0.0f, 0UL, 0UL, 1UL, &outcome ); /* Can't schedule because conflict*/ + schedule_validate_microblock( pack, total_cost_estimate, 0.0f, 0UL, 0UL, 2UL, &outcome ); /* conflict continues ... */ + schedule_validate_microblock( pack, total_cost_estimate, 0.0f, 1UL, 0UL, 0UL, &outcome ); /* conflict gone.*/ } /* The original two that broke my first algorithm */ @@ -307,10 +330,13 @@ void test1( void ) { FD_LOG_NOTICE(( "TEST 1" )); fd_pack_t * pack = init_all( 128UL, 1UL, 128UL, &outcome ); ulong i = 0; - ulong reward1 = make_transaction( i, 500U, 11.0, "A", "B" ); insert( i++, pack ); - ulong reward2 = make_transaction( i, 500U, 10.0, "B", "A" ); insert( i++, pack ); - schedule_validate_microblock( pack, 30000UL, 0.0f, 1UL, reward1, 0UL, &outcome ); - schedule_validate_microblock( pack, 30000UL, 0.0f, 1UL, reward2, 0UL, &outcome ); + ulong cost_estimate; + ulong total_cost_estimate = 0UL; + ulong reward1, reward2; + make_transaction( i, 500U, 500U, 11.0, "A", "B", &reward1, &cost_estimate ); insert( i++, pack ); total_cost_estimate += cost_estimate; + make_transaction( i, 500U, 500U, 10.0, "B", "A", &reward2, &cost_estimate ); insert( i++, pack ); total_cost_estimate += cost_estimate; + schedule_validate_microblock( pack, total_cost_estimate, 0.0f, 1UL, reward1, 0UL, &outcome ); + schedule_validate_microblock( pack, total_cost_estimate, 0.0f, 1UL, reward2, 0UL, &outcome ); } void test2( void ) { @@ -318,12 +344,15 @@ void test2( void ) { fd_pack_t * pack = init_all( 128UL, 1UL, 128UL, &outcome ); ulong i = 0; double j = 13.0; - ulong r0 = make_transaction( i, 500U, j--, "B", "A" ); insert( i++, pack ); - ulong r1 = make_transaction( i, 500U, j--, "C", "B" ); insert( i++, pack ); - ulong r2 = make_transaction( i, 500U, j--, "D", "C" ); insert( i++, pack ); - ulong r3 = make_transaction( i, 500U, j--, "A", "D" ); insert( i++, pack ); - schedule_validate_microblock( pack, 30000UL, 0.0f, 2UL, r0+r2, 0UL, &outcome ); - schedule_validate_microblock( pack, 30000UL, 0.0f, 2UL, r1+r3, 0UL, &outcome ); + ulong cost_estimate; + ulong total_cost_estimate = 0UL; + ulong r0, r1, r2, r3; + make_transaction( i, 500U, 500U, j--, "B", "A", &r0, &cost_estimate ); insert( i++, pack ); total_cost_estimate += cost_estimate; + make_transaction( i, 500U, 500U, j--, "C", "B", &r1, &cost_estimate ); insert( i++, pack ); total_cost_estimate += cost_estimate; + make_transaction( i, 500U, 500U, j--, "D", "C", &r2, &cost_estimate ); insert( i++, pack ); total_cost_estimate += cost_estimate; + make_transaction( i, 500U, 500U, j--, "A", "D", &r3, &cost_estimate ); insert( i++, pack ); total_cost_estimate += cost_estimate; + schedule_validate_microblock( pack, total_cost_estimate, 0.0f, 2UL, r0+r2, 0UL, &outcome ); + schedule_validate_microblock( pack, total_cost_estimate, 0.0f, 2UL, r1+r3, 0UL, &outcome ); /* A smart scheduler that allows read bypass could schedule the first 3 at the same time then #4 after they all finish. */ @@ -333,20 +362,22 @@ void test_vote( void ) { FD_LOG_NOTICE(( "TEST VOTE" )); fd_pack_t * pack = init_all( 128UL, 1UL, 4UL, &outcome ); ulong i = 0; + ulong pack_cost_estimate = 0UL; + uint flags = 0UL; - make_vote_transaction( i ); insert( i++, pack ); - make_vote_transaction( i ); insert( i++, pack ); - make_vote_transaction( i ); insert( i++, pack ); - make_vote_transaction( i ); insert( i++, pack ); + make_vote_transaction( i ); pack_cost_estimate += fd_pack_compute_cost((fd_txn_t const *)txn_scratch[ i ], payload_scratch[ i ], &flags, NULL, NULL, NULL, NULL); insert( i++, pack ); + make_vote_transaction( i ); pack_cost_estimate += fd_pack_compute_cost((fd_txn_t const *)txn_scratch[ i ], payload_scratch[ i ], &flags, NULL, NULL, NULL, NULL); insert( i++, pack ); + make_vote_transaction( i ); pack_cost_estimate += fd_pack_compute_cost((fd_txn_t const *)txn_scratch[ i ], payload_scratch[ i ], &flags, NULL, NULL, NULL, NULL); insert( i++, pack ); + make_vote_transaction( i ); pack_cost_estimate += fd_pack_compute_cost((fd_txn_t const *)txn_scratch[ i ], payload_scratch[ i ], &flags, NULL, NULL, NULL, NULL); insert( i++, pack ); FD_TEST( fd_pack_avail_txn_cnt( pack ) == 4UL ); - schedule_validate_microblock( pack, 30000UL, 0.0f, 0UL, 0UL, 0UL, &outcome ); + schedule_validate_microblock( pack, pack_cost_estimate, 0.0f, 0UL, 0UL, 0UL, &outcome ); FD_TEST( fd_pack_avail_txn_cnt( pack ) == 4UL ); - schedule_validate_microblock( pack, 30000UL, 0.25f, 1UL, 0UL, 0UL, &outcome ); + schedule_validate_microblock( pack, pack_cost_estimate, 0.25f, 1UL, 0UL, 0UL, &outcome ); FD_TEST( fd_pack_avail_txn_cnt( pack ) == 3UL ); - schedule_validate_microblock( pack, 30000UL, 1.0f, 3UL, 0UL, 0UL, &outcome ); + schedule_validate_microblock( pack, pack_cost_estimate, 1.0f, 3UL, 0UL, 0UL, &outcome ); FD_TEST( fd_pack_avail_txn_cnt( pack ) == 0UL ); for( ulong j=0UL; j<3UL; j++ ) FD_TEST( outcome.results[ j ].flags==FD_TXN_P_FLAGS_IS_SIMPLE_VOTE ); @@ -357,13 +388,15 @@ test_delete( void ) { ulong i = 0UL; FD_LOG_NOTICE(( "TEST DELETE" )); fd_pack_t * pack = init_all( 10240UL, 4UL, 128UL, &outcome ); + ulong cost_estimate; + ulong total_cost_estimate = 0UL; - make_transaction( i, 800U, 12.0, "A", "B" ); insert( i++, pack ); - make_transaction( i, 700U, 11.0, "C", "D" ); insert( i++, pack ); - make_transaction( i, 600U, 10.0, "E", "F" ); insert( i++, pack ); - make_transaction( i, 500U, 9.0, "G", "H" ); insert( i++, pack ); - make_transaction( i, 400U, 8.0, "I", "J" ); insert( i++, pack ); - make_transaction( i, 300U, 7.0, "K", "L" ); insert( i++, pack ); + make_transaction( i, 800U, 500U, 12.0, "A", "B", NULL, &cost_estimate ); insert( i++, pack ); total_cost_estimate += cost_estimate; + make_transaction( i, 700U, 500U, 11.0, "C", "D", NULL, &cost_estimate ); insert( i++, pack ); total_cost_estimate += cost_estimate; + make_transaction( i, 600U, 500U, 10.0, "E", "F", NULL, &cost_estimate ); insert( i++, pack ); total_cost_estimate += cost_estimate; + make_transaction( i, 500U, 500U, 9.0, "G", "H", NULL, &cost_estimate ); insert( i++, pack ); total_cost_estimate += cost_estimate; + make_transaction( i, 400U, 500U, 8.0, "I", "J", NULL, &cost_estimate ); insert( i++, pack ); total_cost_estimate += cost_estimate; + make_transaction( i, 300U, 500U, 7.0, "K", "L", NULL, &cost_estimate ); insert( i++, pack ); total_cost_estimate += cost_estimate; FD_TEST( fd_pack_avail_txn_cnt( pack ) == 6UL ); @@ -377,7 +410,7 @@ test_delete( void ) { FD_TEST( fd_pack_avail_txn_cnt( pack ) == 3UL ); - schedule_validate_microblock( pack, 300000UL, 0.0f, 3UL, 0UL, 0UL, &outcome ); + schedule_validate_microblock( pack, total_cost_estimate, 0.0f, 3UL, 0UL, 0UL, &outcome ); FD_TEST( fd_pack_avail_txn_cnt( pack ) == 0UL ); @@ -393,29 +426,32 @@ test_delete( void ) { FD_TEST( fd_pack_avail_txn_cnt( pack ) == 0UL ); i=0UL; - ulong r0 = make_transaction( i, 800U, 12.0, "A", "B" ); insert( i++, pack ); - /* */ make_transaction( i, 700U, 11.0, "A", "D" ); insert( i++, pack ); - ulong r2 = make_transaction( i, 600U, 10.0, "A", "F" ); insert( i++, pack ); - /* */ make_transaction( i, 500U, 9.0, "A", "H" ); insert( i++, pack ); - /* */ make_transaction( i, 400U, 8.0, "A", "J" ); insert( i++, pack ); - /* */ make_transaction( i, 300U, 7.0, "A", "L" ); insert( i++, pack ); + ulong r0, r2; + ulong cost0, cost1, cost5; + make_transaction( i, 800U, 500U, 12.0, "A", "B", &r0, &cost0 ); insert( i++, pack ); total_cost_estimate += cost0; + make_transaction( i, 700U, 500U, 11.0, "A", "D", NULL, &cost1 ); insert( i++, pack ); total_cost_estimate += cost1; + make_transaction( i, 600U, 500U, 10.0, "A", "F", &r2, &cost_estimate ); insert( i++, pack ); total_cost_estimate += cost_estimate; + make_transaction( i, 500U, 500U, 9.0, "A", "H", NULL, &cost_estimate ); insert( i++, pack ); total_cost_estimate += cost_estimate; + make_transaction( i, 400U, 500U, 8.0, "A", "J", NULL, &cost_estimate ); insert( i++, pack ); total_cost_estimate += cost_estimate; + make_transaction( i, 300U, 500U, 7.0, "A", "L", NULL, &cost5 ); insert( i++, pack ); total_cost_estimate += cost5; /* They all conflict now */ - schedule_validate_microblock( pack, 300000UL, 0.0f, 1UL, r0, 1UL, &outcome ); + schedule_validate_microblock( pack, total_cost_estimate, 0.0f, 1UL, r0, 1UL, &outcome ); FD_TEST( fd_pack_avail_txn_cnt( pack ) == 5UL ); + total_cost_estimate -= cost0 + cost1 + cost5; FD_TEST( !fd_pack_delete_transaction( pack, sig0 ) ); FD_TEST( fd_pack_delete_transaction( pack, sig1 ) ); FD_TEST( fd_pack_delete_transaction( pack, sig5 ) ); FD_TEST( fd_pack_avail_txn_cnt( pack ) == 3UL ); /* wait the gap */ - schedule_validate_microblock( pack, 300000UL, 0.0f, 0UL, 0, 2UL, &outcome ); - schedule_validate_microblock( pack, 300000UL, 0.0f, 0UL, 0, 3UL, &outcome ); - schedule_validate_microblock( pack, 300000UL, 0.0f, 0UL, 0, 0UL, &outcome ); + schedule_validate_microblock( pack, total_cost_estimate, 0.0f, 0UL, 0, 2UL, &outcome ); + schedule_validate_microblock( pack, total_cost_estimate, 0.0f, 0UL, 0, 3UL, &outcome ); + schedule_validate_microblock( pack, total_cost_estimate, 0.0f, 0UL, 0, 0UL, &outcome ); - schedule_validate_microblock( pack, 300000UL, 0.0f, 1UL, r2, 1UL, &outcome ); + schedule_validate_microblock( pack, total_cost_estimate, 0.0f, 1UL, r2, 1UL, &outcome ); FD_TEST( fd_pack_delete_transaction( pack, sig3 ) ); FD_TEST( fd_pack_delete_transaction( pack, sig4 ) ); FD_TEST( fd_pack_avail_txn_cnt( pack ) == 0UL ); @@ -427,12 +463,12 @@ test_expiration( void ) { FD_LOG_NOTICE(( "TEST EXPIRATION" )); fd_pack_t * pack = init_all( 10240UL, 4UL, 128UL, &outcome ); - make_transaction( i, 800U, 12.0, "A", "B" ); insert( i++, pack ); - make_transaction( i, 700U, 11.0, "C", "D" ); insert( i++, pack ); - make_transaction( i, 600U, 10.0, "E", "F" ); insert( i++, pack ); - make_transaction( i, 500U, 9.0, "G", "H" ); insert( i++, pack ); - make_transaction( i, 400U, 8.0, "I", "J" ); insert( i++, pack ); - make_transaction( i, 300U, 7.0, "K", "L" ); insert( i++, pack ); + make_transaction( i, 800U, 500U, 12.0, "A", "B", NULL, NULL ); insert( i++, pack ); + make_transaction( i, 700U, 500U, 11.0, "C", "D", NULL, NULL ); insert( i++, pack ); + make_transaction( i, 600U, 500U, 10.0, "E", "F", NULL, NULL ); insert( i++, pack ); + make_transaction( i, 500U, 500U, 9.0, "G", "H", NULL, NULL ); insert( i++, pack ); + make_transaction( i, 400U, 500U, 8.0, "I", "J", NULL, NULL ); insert( i++, pack ); + make_transaction( i, 300U, 500U, 7.0, "K", "L", NULL, NULL ); insert( i++, pack ); FD_TEST( fd_pack_avail_txn_cnt( pack ) == 6UL ); @@ -444,33 +480,33 @@ test_expiration( void ) { FD_TEST( fd_pack_avail_txn_cnt( pack ) == 4UL ); - schedule_validate_microblock( pack, 300000UL, 0.0f, 4UL, 0UL, 0UL, &outcome ); + schedule_validate_microblock( pack, FD_PACK_MAX_COST_PER_BLOCK, 0.0f, 4UL, 0UL, 0UL, &outcome ); FD_TEST( fd_pack_avail_txn_cnt( pack ) == 0UL ); FD_TEST( fd_pack_expire_before( pack, 10UL ) == 0UL ); /* These 4 get rejected because they are expired */ - make_transaction( i, 800U, 12.0, "A", "B" ); insert( i++, pack ); - make_transaction( i, 700U, 11.0, "C", "D" ); insert( i++, pack ); - make_transaction( i, 600U, 10.0, "E", "F" ); insert( i++, pack ); - make_transaction( i, 500U, 9.0, "G", "H" ); insert( i++, pack ); + make_transaction( i, 800U, 500U, 12.0, "A", "B", NULL, NULL ); insert( i++, pack ); + make_transaction( i, 700U, 500U, 11.0, "C", "D", NULL, NULL ); insert( i++, pack ); + make_transaction( i, 600U, 500U, 10.0, "E", "F", NULL, NULL ); insert( i++, pack ); + make_transaction( i, 500U, 500U, 9.0, "G", "H", NULL, NULL ); insert( i++, pack ); FD_TEST( fd_pack_avail_txn_cnt( pack ) == 0UL ); - make_transaction( i, 500U, 9.0, "A", "H" ); insert( i++, pack ); - make_transaction( i, 400U, 8.0, "A", "J" ); insert( i++, pack ); - make_transaction( i, 300U, 7.0, "A", "L" ); insert( i++, pack ); + make_transaction( i, 500U, 500U, 9.0, "A", "H", NULL, NULL ); insert( i++, pack ); + make_transaction( i, 400U, 500U, 8.0, "A", "J", NULL, NULL ); insert( i++, pack ); + make_transaction( i, 300U, 500U, 7.0, "A", "L", NULL, NULL ); insert( i++, pack ); FD_TEST( fd_pack_avail_txn_cnt( pack ) == 3UL ); - schedule_validate_microblock( pack, 300000UL, 0.0f, 1UL, 0UL, 0UL, &outcome ); - schedule_validate_microblock( pack, 300000UL, 0.0f, 0UL, 0UL, 1UL, &outcome ); + schedule_validate_microblock( pack, FD_PACK_MAX_COST_PER_BLOCK, 0.0f, 1UL, 0UL, 0UL, &outcome ); + schedule_validate_microblock( pack, FD_PACK_MAX_COST_PER_BLOCK, 0.0f, 0UL, 0UL, 1UL, &outcome ); FD_TEST( fd_pack_avail_txn_cnt( pack ) == 2UL ); /* Even though txn 10 is expired, it was already scheduled, so account A is still in use. */ FD_TEST( fd_pack_expire_before( pack, 12UL ) == 1UL ); FD_TEST( fd_pack_avail_txn_cnt( pack ) == 1UL ); - schedule_validate_microblock( pack, 300000UL, 0.0f, 0UL, 0UL, 1UL, &outcome ); - schedule_validate_microblock( pack, 300000UL, 0.0f, 1UL, 0UL, 0UL, &outcome ); + schedule_validate_microblock( pack, FD_PACK_MAX_COST_PER_BLOCK, 0.0f, 0UL, 0UL, 1UL, &outcome ); + schedule_validate_microblock( pack, FD_PACK_MAX_COST_PER_BLOCK, 0.0f, 1UL, 0UL, 0UL, &outcome ); FD_TEST( fd_pack_avail_txn_cnt( pack ) == 0UL ); } @@ -480,7 +516,7 @@ performance_test2( void ) { FD_LOG_NOTICE(( "TEST INDEPENDENT PERFORMANCE" )); fd_pack_limits_t limits[ 1 ] = { { - .max_cost_per_block = FD_PACK_MAX_COST_PER_BLOCK, + .max_cost_per_block = 1000000000, .max_vote_cost_per_block = 0UL, .max_write_cost_per_acct = FD_PACK_MAX_WRITE_COST_PER_ACCT, .max_data_bytes_per_block = ULONG_MAX/2UL, @@ -541,7 +577,7 @@ performance_test2( void ) { } ulong scheduled = 0UL; for( ulong i=0UL; i<1024UL/MAX_TXN_PER_MICROBLOCK+1UL; i++ ) { - scheduled += fd_pack_schedule_next_microblock( pack, MAX_TXN_PER_MICROBLOCK*1200UL, 0.0f, i&3UL, outcome.results ); + scheduled += fd_pack_schedule_next_microblock( pack, MAX_TXN_PER_MICROBLOCK*26000UL, 0.0f, i&3UL, outcome.results ); fd_pack_microblock_complete( pack, i&3UL ); } FD_TEST( scheduled==1024UL ); @@ -560,8 +596,9 @@ performance_test2( void ) { void performance_test( int extra_bench ) { ulong i = 0UL; FD_LOG_NOTICE(( "TEST PERFORMANCE" )); - make_transaction( i, 700U, 12.0, "ABC", "DEF" ); /* Total cost 8625 */ - make_transaction( i+1, 500U, 12.0, "GHJ", "KLMNOP" ); /* Total cost 8425 */ + ulong tx1_cost, tx2_cost; + make_transaction( i, 700U, 500U, 12.0, "ABC", "DEF", NULL, &tx1_cost ); /* Total cost 11634 */ + make_transaction( i+1, 500U, 500U, 12.0, "GHJ", "KLMNOP", NULL, &tx2_cost ); /* Total cost 11434 */ fd_wksp_t * wksp = fd_wksp_new_anonymous( FD_SHMEM_GIGANTIC_PAGE_SZ, 1UL, 0UL, "test_pack", 0UL ); @@ -639,9 +676,9 @@ void performance_test( int extra_bench ) { if( FD_LIKELY( iter>=WARMUP ) ) skip0 -= fd_log_wallclock( ); for( ulong j=0UL; j=WARMUP ) ) skip0 += fd_log_wallclock( ); @@ -651,9 +688,9 @@ void performance_test( int extra_bench ) { if( FD_LIKELY( iter>=WARMUP ) ) schedule -= fd_log_wallclock( ); for( ulong j=0UL; j=WARMUP ) ) schedule += fd_log_wallclock( ); @@ -676,11 +713,12 @@ void performance_test( int extra_bench ) { FD_TEST( fd_pack_avail_txn_cnt( pack )==heap_sz ); if( FD_LIKELY( iter>=WARMUP ) ) skip1 -= fd_log_wallclock( ); for( ulong j=0UL; j=WARMUP ) ) skip1 += fd_log_wallclock( ); @@ -750,7 +788,7 @@ void performance_end_block( void ) { ulong footprint = fd_pack_footprint( 4096UL, 0UL, 8UL, limits ); void * _mem = fd_wksp_alloc_laddr( wksp, fd_pack_align(), footprint, 4UL ); - make_transaction( 0UL, 800U, 4.0, "", "" ); + make_transaction( 0UL, 800U, 500U, 4.0, "", "", NULL, NULL ); FD_LOG_NOTICE(( "Writers\tTime (ms/call)" )); fd_pack_t * pack = fd_pack_join( fd_pack_new( _mem, 4096UL, 0UL, 8UL, limits, rng ) ); @@ -796,7 +834,7 @@ void heap_overflow_test( void ) { fd_pack_t * pack = init_all( 1024UL, 1UL, 2UL, &outcome ); /* Insert a bunch of low-paying transactions */ for( ulong j=0UL; j<1024UL; j++ ) { - make_transaction( j, 800U, 4.0, "ABC", "DEF" ); + make_transaction( j, 800U, 500U, 3.0, "ABC", "DEF", NULL, NULL ); /* 11733 cus */ fd_txn_e_t * slot = fd_pack_insert_txn_init( pack ); fd_txn_t * txn = (fd_txn_t*) txn_scratch[ j ]; slot->txnp->payload_sz = payload_sz[ j ]; @@ -810,14 +848,14 @@ void heap_overflow_test( void ) { /* Now insert higher-paying transactions. They should mostly take the place of the low-paying transactions, but it's probabilistic since the transactions conflict a lot. */ - ulong r_hi = make_transaction( 1UL, 500U, 10.0, "GHJ", "KLMNOP" ); + ulong r_hi; for( ulong j=0UL; j<1024UL; j++ ) { - payload_scratch[1][ 1+(j%8) ]++; + make_transaction( j, 500U, 500U, 10.0, "GHJ", "KLMNOP", &r_hi, NULL ); /* 11434 cus */ fd_txn_e_t * slot = fd_pack_insert_txn_init( pack ); - fd_txn_t * txn = (fd_txn_t*) txn_scratch[ 1UL ]; - slot->txnp->payload_sz = payload_sz[ 1UL ]; - fd_memcpy( slot->txnp->payload, payload_scratch[ 1UL ], payload_sz[ 1UL ] ); - fd_memcpy( TXN(slot->txnp), txn, fd_txn_footprint( txn->instr_cnt, txn->addr_table_lookup_cnt ) ); + fd_txn_t * txn = (fd_txn_t*) txn_scratch[ j ]; + slot->txnp->payload_sz = payload_sz[ j ]; + fd_memcpy( slot->txnp->payload, payload_scratch[ j ], payload_sz[ j ] ); + fd_memcpy( TXN(slot->txnp), txn, fd_txn_footprint( txn->instr_cnt, txn->addr_table_lookup_cnt ) ); fd_pack_insert_txn_fini( pack, slot, 0UL ); } @@ -825,7 +863,8 @@ void heap_overflow_test( void ) { FD_TEST( fd_pack_avail_txn_cnt( pack )==1024UL ); for( ulong j=0UL; j<1024UL; j++ ) { - schedule_validate_microblock( pack, 10000UL, 0.0f, j<900UL?1UL:0UL, j<900UL?r_hi:0UL, 0UL, &outcome ); + /* 30000 cannot fit more that 1 transaction. */ + schedule_validate_microblock( pack, 12000, 0.0f, j<900UL?1UL:0UL, j<900UL?r_hi:0UL, 0UL, &outcome ); } FD_TEST( fd_pack_avail_txn_cnt( pack )==0UL ); @@ -837,18 +876,19 @@ test_gap( void ) { for( ulong gap=1UL; gap<=FD_PACK_MAX_BANK_TILES; gap++ ) { fd_pack_t * pack = init_all( 10240UL, gap, 2UL, &outcome ); - ulong i=0UL; - ulong reward1 = make_transaction( i, 500U, 11.0, "A", "B" ); insert( i++, pack ); - ulong reward2 = make_transaction( i, 500U, 10.0, "B", "A" ); insert( i++, pack ); + ulong reward1, reward2; + make_transaction( i, 500U, 500U, 11.0, "A", "B", &reward1, NULL ); insert( i++, pack ); /* 11034 cus */ + make_transaction( i, 500U, 500U, 10.0, "B", "A", &reward2, NULL ); insert( i++, pack ); - schedule_validate_microblock( pack, 10000UL, 0.0f, 1UL, reward1, 0UL, &outcome ); + /* 30000 in only enough to fit 1 transaction */ + schedule_validate_microblock( pack, 12000UL, 0.0f, 1UL, reward1, 0UL, &outcome ); - for( ulong j=1UL; jbank_cu.rebated_cus = outcome.results->pack_cu.requested_execution_cus; + outcome.results->bank_cu.rebated_cus = (uint)((total_cus + (total_cus*FD_PACK_MAX_COST_PER_BLOCK/(4*total_cus))) - FD_PACK_MAX_WRITE_COST_PER_ACCT); fd_pack_rebate_cus( pack, outcome.results, 1UL ); - /* Now consumed CUs is 11,170,954, so it just fits. */ + /* Now consumed CUs is 12M - total_cus, so it just fits. */ schedule_validate_microblock( pack, FD_PACK_MAX_COST_PER_BLOCK, 0.0f, 1UL, 0UL, 0UL, &outcome ); - make_transaction( 0UL, 478310U, 11.0, "A", "B" ); - insert( 0UL, pack ); + make_transaction( j, 10000UL, 500U, 11.0, "A", "B", NULL, NULL ); + insert( j++, pack ); schedule_validate_microblock( pack, FD_PACK_MAX_COST_PER_BLOCK, 0.0f, 0UL, 0UL, 0UL, &outcome ); FD_TEST( fd_pack_avail_txn_cnt( pack )==1UL ); @@ -947,37 +991,42 @@ test_limits( void ) { /* Test the total cost block limit */ if( 1 ) { - fd_pack_t * pack = init_all( 1024UL, 1UL, 1024UL, &outcome ); - /* The limit is based on cost units, and make_transaction takes just - compute CUs. Add the +1 to force the rounding to make these - close enough. */ + fd_pack_t * pack = init_all( 1024UL, 1UL, 512UL, &outcome ); + /* The limit is based on cost units, which are determined by the cost model (i.e. fd_pack_compute_cost). */ + const ulong total_cus = 20334UL; + const ulong almost_full_iter = (FD_PACK_MAX_COST_PER_BLOCK/( 8*total_cus )); ulong i=0UL; - for( ulong j=0UL; jinstr[ 0 ].data_off, 0xFF, 4 ); FD_TEST( insert( i, pack )==FD_PACK_INSERT_REJECT_ESTIMATION_FAIL ); i++; - make_transaction( i, 1000001U, 11.0, "ABC", "DEF" ); /* 6 listed + fee payer + 2 programs */ + make_transaction( i, 1000001U, 500U, 11.0, "ABC", "DEF", NULL, NULL ); /* 6 listed + fee payer + 2 programs */ txn = (fd_txn_t*) txn_scratch[ i ]; txn->addr_table_lookup_cnt = 1; txn->addr_table_adtl_writable_cnt = 20; @@ -1133,11 +1184,11 @@ test_reject( void ) { FD_TEST( insert( i, pack )==FD_PACK_INSERT_REJECT_ACCOUNT_CNT ); i++; - make_transaction( i, 1000001U, 11.0, "A", "A" ); + make_transaction( i, 1000001U, 500U, 11.0, "A", "A", NULL, NULL ); FD_TEST( insert( i, pack )==FD_PACK_INSERT_REJECT_DUPLICATE_ACCT ); i++; - make_transaction( i, 1000001U, 11.0, "A", "B" ); + make_transaction( i, 1000001U, 500U, 11.0, "A", "B", NULL, NULL ); FD_TEST( insert( i, pack )>=0 ); FD_TEST( insert( i, pack )==FD_PACK_INSERT_REJECT_DUPLICATE ); diff --git a/src/flamenco/runtime/fd_runtime.c b/src/flamenco/runtime/fd_runtime.c index 629b3ea72a..79a11f100c 100644 --- a/src/flamenco/runtime/fd_runtime.c +++ b/src/flamenco/runtime/fd_runtime.c @@ -1721,8 +1721,7 @@ fd_runtime_finalize_txn( fd_exec_slot_ctx_t * slot_ctx, } int is_vote = fd_txn_is_simple_vote_transaction( txn_ctx->txn_descriptor, - txn_ctx->_txn_raw->raw, - fd_solana_vote_program_id.key ); + txn_ctx->_txn_raw->raw ); if( !is_vote ){ FD_ATOMIC_FETCH_AND_ADD( &slot_ctx->nonvote_txn_count, 1 ); if( FD_UNLIKELY( exec_txn_err ) ){ diff --git a/src/flamenco/runtime/tests/fd_pack_test.c b/src/flamenco/runtime/tests/fd_pack_test.c index 85801ce9d8..5ac9fd3a06 100644 --- a/src/flamenco/runtime/tests/fd_pack_test.c +++ b/src/flamenco/runtime/tests/fd_pack_test.c @@ -41,11 +41,12 @@ do { } ulong rewards; uint compute_unit_limit; + ulong loaded_accounts_data_cost = 0UL; fd_compute_budget_program_finalize( cbp_state, input->instr_datas_count, &rewards, - &compute_unit_limit - ); + &compute_unit_limit, + &loaded_accounts_data_cost ); effects->rewards = rewards; effects->compute_unit_limit = compute_unit_limit;