Skip to content

Commit

Permalink
[short][move e2e bench] Add gas metrics and more existing transaction…
Browse files Browse the repository at this point in the history 
… types (#15302)

## Description
This allows for easier understanding of gas calibration. Looking at gas/s metrics (and execution and io breakdown) makes it easy to calibrate new constants.

## How Has This Been Tested?
used to calibrate vector::move_range native
  • Loading branch information
@igor-aptos
igor-aptos authored Nov 19, 2024
1 parent 11c4819 commit 6acc3d5
Show file tree
Hide file tree
Showing 7 changed files with 164 additions and 89 deletions.
20 changes: 11 additions & 9 deletions aptos-move/aptos-gas-calibration/src/measurements.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,15 +162,17 @@ fn compile_and_run_samples_ir(
.equation_names
.push(format!("{}::{}", &identifier, func_identifier.0));

let elapsed = executor.exec_func_record_running_time(
&module_id,
&func_identifier.0,
vec![],
func_identifier.1.clone(),
iterations,
ExecFuncTimerDynamicArgs::NoArgs,
GasMeterType::UnmeteredGasMeter,
);
let elapsed = executor
.exec_func_record_running_time(
&module_id,
&func_identifier.0,
vec![],
func_identifier.1.clone(),
iterations,
ExecFuncTimerDynamicArgs::NoArgs,
GasMeterType::UnmeteredGasMeter,
)
.elapsed_micros();
gas_measurement.regular_meter.push(elapsed);

// record with abstract gas meter
Expand Down
20 changes: 11 additions & 9 deletions aptos-move/aptos-gas-calibration/src/measurements_helpers.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,15 +94,17 @@ pub fn execute_user_txn(
iterations: u64,
args: Vec<Vec<u8>>,
) -> u128 {
let elapsed = executor.exec_func_record_running_time(
module_name,
function_name,
vec![],
args,
iterations,
ExecFuncTimerDynamicArgs::NoArgs,
GasMeterType::UnmeteredGasMeter,
);
let elapsed = executor
.exec_func_record_running_time(
module_name,
function_name,
vec![],
args,
iterations,
ExecFuncTimerDynamicArgs::NoArgs,
GasMeterType::UnmeteredGasMeter,
)
.elapsed_micros();
println!("running time (microseconds): {}", elapsed);
elapsed
}
Expand Down
20 changes: 11 additions & 9 deletions aptos-move/aptos-vm-benchmarks/src/helper.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,15 +64,17 @@ pub fn execute_module_txn(

// sign user transaction and only records the body of the transaction
pub fn execute_user_txn(executor: &mut FakeExecutor, module_name: &ModuleId, function_name: &str) {
let elapsed = executor.exec_func_record_running_time(
module_name,
function_name,
vec![],
vec![],
10,
ExecFuncTimerDynamicArgs::NoArgs,
GasMeterType::UnmeteredGasMeter,
);
let elapsed = executor
.exec_func_record_running_time(
module_name,
function_name,
vec![],
vec![],
10,
ExecFuncTimerDynamicArgs::NoArgs,
GasMeterType::UnmeteredGasMeter,
)
.elapsed_micros();
println!("running time (microseconds): {}", elapsed);
}

Expand Down
96 changes: 56 additions & 40 deletions aptos-move/e2e-benchmark/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

use aptos_language_e2e_tests::{
account::Account,
executor::{ExecFuncTimerDynamicArgs, FakeExecutor, GasMeterType},
executor::{ExecFuncTimerDynamicArgs, FakeExecutor, GasMeterType, Measurement},
};
use aptos_transaction_generator_lib::{
publishing::{
Expand Down Expand Up @@ -46,7 +46,7 @@ fn execute_and_time_entry_point(
publisher_address: &AccountAddress,
executor: &mut FakeExecutor,
iterations: u64,
) -> u128 {
) -> Measurement {
let mut rng = StdRng::seed_from_u64(14);
let entry_fun = entry_point
.create_payload(
Expand Down Expand Up @@ -77,9 +77,9 @@ fn execute_and_time_entry_point(
)
}

const ALLOWED_REGRESSION: f32 = 0.15;
const ALLOWED_IMPROVEMENT: f32 = 0.15;
const ABSOLUTE_BUFFER_US: f32 = 2.0;
const ALLOWED_REGRESSION: f64 = 0.15;
const ALLOWED_IMPROVEMENT: f64 = 0.15;
const ABSOLUTE_BUFFER_US: f64 = 2.0;

const CALIBRATION_VALUES: &str = "
Loop { loop_count: Some(100000), loop_type: NoOp } 6 0.988 1.039 41212.4
Expand All @@ -88,9 +88,9 @@ CreateObjects { num_objects: 10, object_payload_size: 0 } 6 0.940 1.026 152.1
CreateObjects { num_objects: 10, object_payload_size: 10240 } 6 0.934 1.051 9731.3
CreateObjects { num_objects: 100, object_payload_size: 0 } 6 0.966 1.051 1458.3
CreateObjects { num_objects: 100, object_payload_size: 10240 } 6 0.969 1.077 11196.4
InitializeVectorPicture { length: 40 } 6 0.973 1.066 75.0
VectorPicture { length: 40 } 6 0.955 1.092 22.0
VectorPictureRead { length: 40 } 6 0.952 1.047 21.0
InitializeVectorPicture { length: 128 } 6 0.973 1.066 170.3
VectorPicture { length: 128 } 6 0.955 1.092 46.2
VectorPictureRead { length: 128 } 6 0.952 1.047 45.1
InitializeVectorPicture { length: 30720 } 6 0.969 1.071 27295.8
VectorPicture { length: 30720 } 6 0.957 1.066 6560.2
VectorPictureRead { length: 30720 } 6 0.948 1.053 6642.8
Expand All @@ -103,11 +103,16 @@ TokenV1MintAndTransferNFTSequential 6 0.991 1.067 543.7
TokenV2AmbassadorMint { numbered: true } 6 0.987 1.052 474.4
LiquidityPoolSwap { is_stable: true } 6 0.970 1.042 555.4
LiquidityPoolSwap { is_stable: false } 6 0.925 1.001 535.3
CoinInitAndMint 6 0.925 1.001 197.1
FungibleAssetMint 6 0.925 1.001 231.6
IncGlobalMilestoneAggV2 { milestone_every: 1 } 6 0.925 1.001 33.3
IncGlobalMilestoneAggV2 { milestone_every: 2 } 6 0.925 1.001 19.1
EmitEvents { count: 1000 } 6 0.925 1.001 8493.7
";

struct CalibrationInfo {
// count: usize,
expected_time: f32,
expected_time_micros: f64,
}

fn get_parsed_calibration_values() -> HashMap<String, CalibrationInfo> {
Expand All @@ -118,7 +123,7 @@ fn get_parsed_calibration_values() -> HashMap<String, CalibrationInfo> {
let parts = line.split('\t').collect::<Vec<_>>();
(parts[0].to_string(), CalibrationInfo {
// count: parts[1].parse().unwrap(),
expected_time: parts[parts.len() - 1].parse().unwrap(),
expected_time_micros: parts[parts.len() - 1].parse().unwrap(),
})
})
.collect()
Expand Down Expand Up @@ -146,7 +151,7 @@ fn main() {
loop_type: LoopType::Arithmetic,
},
// This is a cheap bcs (serializing vec<u8>), so not representative of what BCS native call should cost.
// (, EntryPoints::Loop { loop_count: Some(1000), loop_type: LoopType::BCS { len: 1024 }}),
// (, EntryPoints::Loop { loop_count: Some(1000), loop_type: LoopType::BcsToBytes { len: 1024 }}),
EntryPoints::CreateObjects {
num_objects: 10,
object_payload_size: 0,
Expand All @@ -163,9 +168,9 @@ fn main() {
num_objects: 100,
object_payload_size: 10 * 1024,
},
EntryPoints::InitializeVectorPicture { length: 40 },
EntryPoints::VectorPicture { length: 40 },
EntryPoints::VectorPictureRead { length: 40 },
EntryPoints::InitializeVectorPicture { length: 128 },
EntryPoints::VectorPicture { length: 128 },
EntryPoints::VectorPictureRead { length: 128 },
EntryPoints::InitializeVectorPicture { length: 30 * 1024 },
EntryPoints::VectorPicture { length: 30 * 1024 },
EntryPoints::VectorPictureRead { length: 30 * 1024 },
Expand All @@ -188,22 +193,27 @@ fn main() {
EntryPoints::TokenV2AmbassadorMint { numbered: true },
EntryPoints::LiquidityPoolSwap { is_stable: true },
EntryPoints::LiquidityPoolSwap { is_stable: false },
EntryPoints::CoinInitAndMint,
EntryPoints::FungibleAssetMint,
EntryPoints::IncGlobalMilestoneAggV2 { milestone_every: 1 },
EntryPoints::IncGlobalMilestoneAggV2 { milestone_every: 2 },
EntryPoints::EmitEvents { count: 1000 },
];

let mut failures = Vec::new();
let mut json_lines = Vec::new();

println!(
"{:>15} {:>15} {:>15} entry point",
"wall time (us)", "expected (us)", "diff(- is impr)"
"{:>13} {:>13} {:>13}{:>13} {:>13} {:>13} entry point",
"walltime(us)", "expected(us)", "dif(- is impr)", "gas/s", "exe gas", "io gas",
);

for (index, entry_point) in entry_points.into_iter().enumerate() {
let entry_point_name = format!("{:?}", entry_point);
let expected_time = calibration_values
let expected_time_micros = calibration_values
.get(&entry_point_name)
.unwrap()
.expected_time;
.expect(&entry_point_name)
.expected_time_micros;
let publisher = executor.new_account_at(AccountAddress::random());

let mut package_handler = PackageHandler::new(entry_point.package_name());
Expand All @@ -215,7 +225,6 @@ fn main() {
0,
package.publish_transaction_payload(),
);
println!("Published package: {:?}", entry_point.package_name());
if let Some(init_entry_point) = entry_point.initialize_entry_point() {
execute_txn(
&mut executor,
Expand All @@ -228,52 +237,59 @@ fn main() {
Some(publisher.address()),
),
);
println!(
"Executed init entry point: {:?}",
entry_point.initialize_entry_point()
);
}

let elapsed_micros = execute_and_time_entry_point(
let measurement = execute_and_time_entry_point(
&entry_point,
&package,
publisher.address(),
&mut executor,
if expected_time > 10000.0 {
if expected_time_micros > 10000.0 {
6
} else if expected_time > 1000.0 {
} else if expected_time_micros > 1000.0 {
10
} else {
100
},
);
let diff = (elapsed_micros as f32 - expected_time) / expected_time * 100.0;
let elapsed_micros = measurement.elapsed_micros_f64();
let diff = (elapsed_micros - expected_time_micros) / expected_time_micros * 100.0;
let execution_gas_units = measurement.execution_gas_units();
let io_gas_units = measurement.io_gas_units();
let gps = (execution_gas_units + io_gas_units) / measurement.elapsed_secs_f64();
println!(
"{:15} {:15.1} {:14.1}% {:?}",
elapsed_micros, expected_time, diff, entry_point
"{:13.1} {:13.1} {:12.1}% {:13.0} {:13.2} {:13.2} {:?}",
elapsed_micros,
expected_time_micros,
diff,
gps,
execution_gas_units,
io_gas_units,
entry_point
);

json_lines.push(json!({
"grep": "grep_json_aptos_move_vm_perf",
"transaction_type": entry_point_name,
"wall_time_us": elapsed_micros,
"expected_wall_time_us": expected_time,
"gas_units_per_second": gps,
"execution_gas_units": execution_gas_units,
"io_gas_units": io_gas_units,
"expected_wall_time_us": expected_time_micros,
"test_index": index,
}));

if elapsed_micros as f32
> expected_time as f32 * (1.0 + ALLOWED_REGRESSION) + ABSOLUTE_BUFFER_US
{
if elapsed_micros > expected_time_micros * (1.0 + ALLOWED_REGRESSION) + ABSOLUTE_BUFFER_US {
failures.push(format!(
"Performance regression detected: {}us, expected: {}us, diff: {}%, for {:?}",
elapsed_micros, expected_time, diff, entry_point
"Performance regression detected: {:.1}us, expected: {:.1}us, diff: {}%, for {:?}",
elapsed_micros, expected_time_micros, diff, entry_point
));
} else if elapsed_micros as f32 + ABSOLUTE_BUFFER_US
< expected_time as f32 * (1.0 - ALLOWED_IMPROVEMENT)
} else if elapsed_micros + ABSOLUTE_BUFFER_US
< expected_time_micros * (1.0 - ALLOWED_IMPROVEMENT)
{
failures.push(format!(
"Performance improvement detected: {}us, expected {}us, diff: {}%, for {:?}. You need to adjust expected time!",
elapsed_micros, expected_time, diff, entry_point
"Performance improvement detected: {:.1}us, expected {:.1}us, diff: {}%, for {:?}. You need to adjust expected time!",
elapsed_micros, expected_time_micros, diff, entry_point
));
}
}
Expand Down
Loading

0 comments on commit 6acc3d5

Please sign in to comment.