diff --git a/discord.sh b/discord.sh index dc21a63..023b9be 100755 --- a/discord.sh +++ b/discord.sh @@ -33,8 +33,8 @@ printf -v s_ct_stats_number_of_accts '%s\\n%s\\n%s\\n%s' \ "$mean_ct_stats_num_of_accts_txt" "$max_ct_stats_num_of_accts_txt" "$p90_ct_stats_num_of_accts_txt" "$p99_ct_stats_num_of_accts_txt" printf -v blocks_fill '%s\\n%s\\n%s\\n%s\\n%s' \ "$total_blocks_txt" "$blocks_fill_50_txt" "$blocks_fill_90_txt" "$blocks_fill_50_percent_txt" "$blocks_fill_90_percent_txt" -printf -v skip_rate '%s\\n%s\\n%s\\n' \ - "$mean_skip_rate_txt" "$max_skip_rate_txt" "$skip_rate_90_txt" +printf -v skip_rate '%s\\n%s\\n%s\\n%s\\n' \ + "$mean_skip_rate_txt" "$max_skip_rate_txt" "$skip_rate_90_txt" "$mean_skip_rate_b4_test_txt" printf -v buildkite_link '%s' "[Buildkite]($BUILDKITE_BUILD_URL)" printf -v grafana_link '%s' "[Grafana]($gf_url)" diff --git a/dos-report.sh b/dos-report.sh index 8a5f34f..9586e73 100755 --- a/dos-report.sh +++ b/dos-report.sh @@ -53,6 +53,7 @@ stop_time=$STOP_TIME stop_time2=$STOP_TIME2 ## make sure +source utils.sh source influx_data.sh query(){ @@ -334,9 +335,15 @@ DATAPOINT[max_skip_rate]="$_value" result_input="${FLUX_RESULT['skip_rate_90']}" get_value [[ $_value != "na" ]] && printf -v precision "%.2f" "$_value" || precision="na" -skip_rate_90_txt="skip_rate_90_txt: $precision%" +skip_rate_90_txt="skip_rate_90: $precision%" DATAPOINT[skip_rate_90]="$_value" +result_input="${FLUX_RESULT['mean_skip_rate_b4_test']}" +get_value +[[ $_value != "na" ]] && printf -v precision "%.2f" "$_value" || precision="na" +mean_skip_rate_b4_test_txt="mean_skip_rate_b4_test: $precision%" +DATAPOINT[mean_skip_rate_b4_test]="$_value" + #write data report to the influx build="$BUILDKITE_BUILD_NUMBER" diff --git a/influx_data.sh b/influx_data.sh index 04e8019..0c2cb31 100755 --- a/influx_data.sh +++ b/influx_data.sh @@ -209,17 +209,22 @@ _blocks_fill_90='from(bucket: "tds")|> range(start:'${start_time}' ,stop:'${stop |> sum(column: "bank_slot") |> drop(columns: ["_start", "_stop"])' #skip_rate -_mean_skip_rate='data_max=from(bucket: "tds")|> range(start:'${start_time}' ,stop:'${stop_time}') +# $1:start_time +# $2: stop_time +# $3: oversize_window +# $4: type of statistic (mean/max/percentile90) +function skip_rate_query() { + skip_rate_q_prefix='data_max=from(bucket: "tds")|> range(start:'$1' ,stop:'$2') |> filter(fn: (r) => r["_measurement"] == "bank-new_from_parent-heights") |> filter(fn: (r) => r["_field"] == "slot" or r["_field"] == "block_height") - |> aggregateWindow(every:'${oversize_window}', fn:max) + |> aggregateWindow(every:'$3', fn:max) |> max() |> group(columns: ["host_id"], mode:"by") data_min=from(bucket: "tds") - |> range(start:'${start_time}' ,stop:'${stop_time}') + |> range(start:'$1' ,stop:'$2') |> filter(fn: (r) => r["_measurement"] == "bank-new_from_parent-heights") |> filter(fn: (r) => r["_field"] == "slot" or r["_field"] == "block_height") - |> aggregateWindow(every: '${oversize_window}', fn:min) + |> aggregateWindow(every: '$3', fn:min) |> max() |> group(columns: ["host_id"], mode:"by") block_max=data_max|> filter(fn: (r) => r["_field"] == "block_height")|> set(key: "_field", value: "block_max") @@ -233,58 +238,29 @@ _mean_skip_rate='data_max=from(bucket: "tds")|> range(start:'${start_time}' ,sto |> map(fn: (r) => ({ r with skip_slot: r.slot_diff - r.block_diff })) |> filter(fn: (r) => r.slot_diff > 0) |> map(fn: (r) => ({ r with skip_rate_percent: r.skip_slot*100/r.slot_diff })) - |> keep(columns: ["skip_rate_percent"])|> group()|> mean(column:"skip_rate_percent")' -_max_skip_rate='data_max=from(bucket: "tds")|> range(start:'${start_time}' ,stop:'${stop_time}') - |> filter(fn: (r) => r["_measurement"] == "bank-new_from_parent-heights") - |> filter(fn: (r) => r["_field"] == "slot" or r["_field"] == "block_height") - |> aggregateWindow(every:'${oversize_window}', fn:max) - |> max() - |> group(columns: ["host_id"], mode:"by") - data_min=from(bucket: "tds") - |> range(start:'${start_time}' ,stop:'${stop_time}') - |> filter(fn: (r) => r["_measurement"] == "bank-new_from_parent-heights") - |> filter(fn: (r) => r["_field"] == "slot" or r["_field"] == "block_height") - |> aggregateWindow(every: '${oversize_window}', fn:min) - |> max() - |> group(columns: ["host_id"], mode:"by") - block_max=data_max|> filter(fn: (r) => r["_field"] == "block_height")|> set(key: "_field", value: "block_max") - block_min=data_min|> filter(fn: (r) => r["_field"] == "block_height")|> set(key: "_field", value: "block_min") - slot_max=data_max|> filter(fn: (r) => r["_field"] == "slot")|> set(key: "_field", value: "slot_max") - slot_min=data_min|> filter(fn: (r) => r["_field"] == "slot")|> set(key: "_field", value: "slot_min") - union(tables: [block_max, block_min, slot_max, slot_min]) - |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") - |> map(fn: (r) => ({ r with block_diff: r.block_max - r.block_min })) - |> map(fn: (r) => ({ r with slot_diff: r.slot_max - r.slot_min })) - |> map(fn: (r) => ({ r with skip_slot: r.slot_diff - r.block_diff })) - |> filter(fn: (r) => r.slot_diff > 0) - |> map(fn: (r) => ({ r with skip_rate_percent: r.skip_slot*100/r.slot_diff })) - |> keep(columns: ["skip_rate_percent"])|> group()|> max(column:"skip_rate_percent")' - -_skip_rate_90='data_max=from(bucket: "tds")|> range(start:'${start_time}' ,stop:'${stop_time}') - |> filter(fn: (r) => r["_measurement"] == "bank-new_from_parent-heights") - |> filter(fn: (r) => r["_field"] == "slot" or r["_field"] == "block_height") - |> aggregateWindow(every:'${oversize_window}', fn:max) - |> max() - |> group(columns: ["host_id"], mode:"by") - data_min=from(bucket: "tds") - |> range(start:'${start_time}' ,stop:'${stop_time}') - |> filter(fn: (r) => r["_measurement"] == "bank-new_from_parent-heights") - |> filter(fn: (r) => r["_field"] == "slot" or r["_field"] == "block_height") - |> aggregateWindow(every: '${oversize_window}', fn:min) - |> max() - |> group(columns: ["host_id"], mode:"by") - block_max=data_max|> filter(fn: (r) => r["_field"] == "block_height")|> set(key: "_field", value: "block_max") - block_min=data_min|> filter(fn: (r) => r["_field"] == "block_height")|> set(key: "_field", value: "block_min") - slot_max=data_max|> filter(fn: (r) => r["_field"] == "slot")|> set(key: "_field", value: "slot_max") - slot_min=data_min|> filter(fn: (r) => r["_field"] == "slot")|> set(key: "_field", value: "slot_min") - union(tables: [block_max, block_min, slot_max, slot_min]) - |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") - |> map(fn: (r) => ({ r with block_diff: r.block_max - r.block_min })) - |> map(fn: (r) => ({ r with slot_diff: r.slot_max - r.slot_min })) - |> map(fn: (r) => ({ r with skip_slot: r.slot_diff - r.block_diff })) - |> filter(fn: (r) => r.slot_diff > 0) - |> map(fn: (r) => ({ r with skip_rate_percent: r.skip_slot*100/r.slot_diff })) - |> keep(columns: ["skip_rate_percent"])|> group()|> quantile(column: "skip_rate_percent", q: 0.90)' + |> keep(columns: ["skip_rate_percent"])|> group()' + case "$4" in + 'mean') + skip_rate_query=$skip_rate_q_prefix'|> mean(column: "skip_rate_percent")' + ;; + 'max') + skip_rate_query=$skip_rate_q_prefix'|> max(column: "skip_rate_percent")' + ;; + 'percentile90') + skip_rate_query=$skip_rate_q_prefix'|> quantile(q: 0.9, column: "skip_rate_percent")' + ;; + esac +} +skip_rate_query "$start_time" "$stop_time" "$oversize_window" "mean" +_mean_skip_rate=$skip_rate_query +skip_rate_query "$start_time" "$stop_time" "$oversize_window" "max" +_max_skip_rate=$skip_rate_query +skip_rate_query "$start_time" "$stop_time" "$oversize_window" "percentile90" +_skip_rate_90=$skip_rate_query +start_time_b4_test=$(get_time_before "$start_time" 3600) +b4_stop_time_b4_test="$start_time" +skip_rate_query "$start_time_b4_test" "$b4_stop_time_b4_test" "$oversize_window" "mean" +_mean_skip_rate_b4_test=$skip_rate_query declare -A FLUX # FLUX command FLUX[start_slot]=$_start_slot @@ -336,6 +312,7 @@ FLUX[blocks_fill_90]=$_blocks_fill_90 FLUX[mean_skip_rate]=$_mean_skip_rate FLUX[max_skip_rate]=$_max_skip_rate FLUX[skip_rate_90]=$_skip_rate_90 +FLUX[mean_skip_rate_b4_test]=$_mean_skip_rate_b4_test # Dos Report write to Influxdb @@ -386,4 +363,5 @@ FIELD_MEASUREMENT[blocks_90_full]=block_fill # skip rate FIELD_MEASUREMENT[mean_skip_rate]=skip_rate FIELD_MEASUREMENT[max_skip_rate]=skip_rate -FIELD_MEASUREMENT[skip_rate_90]=skip_rate \ No newline at end of file +FIELD_MEASUREMENT[skip_rate_90]=skip_rate +FIELD_MEASUREMENT[mean_skip_rate_b4_test]=skip_rate