Review Workload pipeline CSV parser methods #5068
Comments
UpdateI created a script to test the CSV parser functionality and generated the statistics for the following artifacts. The results seem to be correct, I will continue to investigate any issues I may be missing. Scriptimport argparse
from wazuh_testing.tools.performance.csv_parser import ClusterCSVTasksParser, ClusterCSVResourcesParser
def get_script_arguments():
parser = argparse.ArgumentParser(usage="%(prog)s [options]", description="Script to generate data visualizations",
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('-a', '--artifacts_path', dest='artifacts_path', required=True, type=str,
action='store', help='Directory where the cluster CSVs can be found.')
return parser.parse_args()
def main():
options = get_script_arguments()
tasks = ClusterCSVTasksParser(artifacts_path=options.artifacts_path)
print(tasks.get_stats())
resources = ClusterCSVResourcesParser(artifacts_path=options.artifacts_path)
print(resources.get_stats())
if __name__ == '__main__':
main()Tasks{
"setup_phase": {
"integrity_sync": {
"time_spent(s)": {
"workers": {
"mean":
("worker_2", 0.6762659145850121),
"max":
("worker_2", 12.629)
},
"master": {
"mean":
("master", 0.6762659145850121),
"max":
("master", 12.629)
}
}
},
"agent-info_sync": {
"time_spent(s)": {
"workers": {
"mean":
("worker_4", 0.8725),
"max":
("worker_22", 14.119)
},
"master": {
"mean":
("master", 0.6845116607773851),
"max":
("master", 12.945)
}
}
},
"integrity_check": {
"time_spent(s)": {
"workers": {
"mean":
("worker_9", 0.42400746268656714),
"max":
("worker_22", 14.419)
},
"master": {
"mean":
("master", 0.06794084084084084),
"max":
("master", 1.148)
}
}
}
},
"stable_phase": {
"agent-info_sync": {
"time_spent(s)": {
"workers": {
"mean":
("worker_2", 0.14200000000000002),
"max":
("worker_2", 0.251)
},
"master": {
"mean":
("master", 0.14200000000000002),
"max":
("master", 0.251)
}
}
},
"integrity_check": {
"time_spent(s)": {
"workers": {
"mean":
("worker_22", 0.0131304347826087),
"max":
("worker_8", 0.042)
},
"master": {
"mean":
("master", 0.004652727272727274),
"max":
("master", 0.022)
}
}
}
}
}Resources{
"setup_phase": {
"wazuh-clusterd": {
"USS(KB)": {
"workers": {
"mean":
("worker_4", 53349.6),
"max":
("worker_1", 80688.0),
"reg_cof":
("worker_2", 179.27511093704192)
},
"master": {
"mean":
("master", 62363.29770992367),
"max":
("master", 95100.0),
"reg_cof":
("master", 91.86956877118291)
}
},
"CPU(%)": {
"workers": {
"mean":
("worker_13", 1.3464684014869888),
"max":
("worker_6", 12.4),
"reg_cof":
("worker_6", -0.0037710241657507556)
},
"master": {
"mean":
("master", 33.14236641221375),
"max":
("master", 143.5),
"reg_cof":
("master", -0.18340651315418535)
}
},
"FD": {
"workers": {
"mean":
("worker_2", 73.23255813953489),
"max":
("worker_25", 76.0),
"reg_cof":
("worker_2", -0.0072866796240246366)
},
"master": {
"mean":
("master", 130.6030534351145),
"max":
("master", 161.0),
"reg_cof":
("master", -0.0478306111507564)
}
}
}
},
"stable_phase": {
"wazuh-clusterd": {
"USS(KB)": {
"workers": {
"mean":
("worker_2", 74357.9512195122),
"max":
("worker_2", 75844.0),
"reg_cof":
("worker_2", 208.56585365853752)
},
"master": {
"mean":
("master", 62881.46341463415),
"max":
("master", 62940.0),
"reg_cof":
("master", 3.854355400696879)
}
},
"CPU(%)": {
"workers": {
"mean":
("worker_2", 0.8097560975609757),
"max":
("worker_2", 16.0),
"reg_cof":
("worker_14", 0.001393728222996515)
},
"master": {
"mean":
("master", 1.9365853658536583),
"max":
("master", 8.2),
"reg_cof":
("master", -0.042822299651567884)
}
},
"FD": {
"workers": {
"mean":
("worker_25", 73.0),
"max":
("worker_17", 73.0),
"reg_cof":
("worker_23", 0.10480217456961556)
},
"master": {
"mean":
("master", 133.0),
"max":
("master", 133.0),
"reg_cof":
("master", 4.1492513876640575e-16)
}
}
}
}
} |
UpdateI tested the script against the artifacts that caused this issue to be opened and I could find that the reason why the values are lower is because the timestamps between the parent process ( The files have 313, 314 and 314 lines, whereas the concatenated one has 560 (there's 246 timestamp mismatches). This caused the mean and max values to not be accurate since their calculations were done with the sum of 1 (parent process) and in other cases 2 (child processes) values. Considering this and the fact that the tests performed on other artifacts were successful, I conclude that the issue has to be with those specific artifacts having different timestamps instead of the script itself. Artifacts: 50000A25W.zip Before grouping, concatenation and sumListwazuh-clusterd
0 57
1 57
2 57
3 57
4 57
..
308 58
309 59
310 61
311 62
312 61
Name: FD, Length: 313, dtype: int64
wazuh-clusterd_child_1
0 29
1 29
2 29
3 29
4 29
..
309 29
310 29
311 29
312 30
313 29
Name: FD, Length: 314, dtype: int64
wazuh-clusterd_child_2
0 31
1 31
2 31
3 31
4 31
..
309 31
310 31
311 32
312 31
313 31
Name: FD, Length: 314, dtype: int64After grouping, concatenation and sumListFull listTimestamps differences
|
ReviewThe analysis was correctly carried out, taking a deeper look at the data that generated the misbehavior. The CSV parser methods group by the |
UpdateA potential solution could be to group the values by ranges instead of a single value, although it won't be guaranteed that all the ranges would be made up of the same number of values and this could alter the results. The only value that the files share and that distinguishes each of the rows is the timestamp, so if the solution mentioned above is not useful, the way in which execution times are evaluated would have to change, potentially dropping use of this script altogether. Considering this and that the bug occurred in version 4.3.0 and has not occurred since, I think it would be appropriate to investigate a solution in a separate issue with a different priority. |
ReviewI agree with the last Update, the alternative solution should take place in a separate issue and we must keep the current behavior in mind when reviewing the output metrics. LGTM! |
Final reviewGood analysis @GGP1. I have also taken a look at recent artifacts, and as you say, it seems that it is not common to find different timestamps. What I have seen is that the number of rows is different sometimes, probably due to the fact that the API Performance tests restart the master and then the entire cluster. These cases do not seem worrying since the existing rows do maintain the same timestamp. The only thing that could cause the timestamps to be different, as in the artifacts you mentioned here, is if some of the threads in the script take longer than the rest to obtain metrics. It only needs to happen once for the timestamps to no longer match: wazuh-qa/deps/wazuh_testing/wazuh_testing/tools/performance/binary.py Lines 200 to 209 in 5061544 A possible alternative solution could be that, instead of running the loop every n seconds, do it in UTC time blocks. For example, every 5 seconds:
I agree with you in opening a new issue to study possible fixes for this. The priority will depend on the frequency with which we encounter these problems in the future, but it doesn't seem a common behavior. |
Description
During wazuh/wazuh#22179, it was found that some of the set thresholds for the resource measurement comparisons were wrongly obtained due to the misbehavior of the CSV Parser module (found in
deps/wazuh_testing/wazuh_testing/tools/performance/csv_parser.py). Using the values from a Workload test near the date that the threshold was established, the values (especially the ones for the File Descriptors), seem to be correct:{ "setup_phase": { "wazuh-clusterd": { "USS(KB)": { "workers": { "mean": ("worker_23", 79302.4262295082), "max": ("worker_16", 157676.0), "reg_cof": ("worker_23", 887.0418113746235) }, "master": { "mean": ("master", 127014.2972972973), "max": ("master", 267488.0), "reg_cof": ("master", 596.5368609261227) } }, "CPU(%)": { "workers": { "mean": ("worker_14", 9.087704918032788), "max": ("worker_14", 50.699999999999996), "reg_cof": ("worker_14", 0.11926369947888614) }, "master": { "mean": ("master", 72.27432432432431), "max": ("master", 179.8), "reg_cof": ("master", 0.37989057404206) } }, "FD": { "workers": { "mean": ("worker_14", 63.122950819672134), "max": ("worker_1", 65), "reg_cof": ("worker_24", 0.04868202764976976) }, "master": { "mean": ("master", 98.82432432432432), "max": ("master", 123), "reg_cof": ("master", -0.31726124151819285) } } } }, "stable_phase": { "wazuh-clusterd": { "USS(KB)": { "workers": { "mean": ("worker_11", 116553.51381215469), "max": ("worker_13", 205164.0), "reg_cof": ("worker_11", 10.248787385086283) }, "master": { "mean": ("master", 141715.49247311827), "max": ("master", 267820.0), "reg_cof": ("master", -117.77925544357818) } }, "CPU(%)": { "workers": { "mean": ("worker_17", 10.236627906976745), "max": ("worker_10", 28.4), "reg_cof": ("worker_11", -0.0028954773944346317) }, "master": { "mean": ("master", 57.59978494623656), "max": ("master", 118.4), "reg_cof": ("master", -0.050952369585662786) } }, "FD": { "workers": { "mean": ("worker_17", 54.61046511627907), "max": ("worker_1", 64), "reg_cof": ("worker_11", -0.0159273770496572) }, "master": { "mean": ("master", 75.8752688172043), "max": ("master", 121), "reg_cof": ("master", -0.0680563048117365) } } } } }Values from the issue that established the latest threshold values:
{ "setup_phase": { "wazuh-clusterd": { "USS(KB)": { "workers": { "mean": ("worker_10", 85625.87368421053), "max": ("worker_14", 170588.0), "reg_cof": ("worker_10", 479.74241012653823) }, "master": { "mean": ("master", 196930.5723905724), "max": ("master", 542408.0), "reg_cof": ("master", 159.20029975063525) } }, "CPU(%)": { "workers": { "mean": ("worker_10", 7.013684210526315), "max": ("worker_15", 37.4), "reg_cof": ("worker_8", 0.03610734796000005) }, "master": { "mean": ("master", 46.47138047138047), "max": ("master", 96.9), "reg_cof": ("master", 0.04855447271554653) } }, "FD": { "workers": { "mean": ("worker_10", 63.28947368421053), "max": ("worker_1", 65), "reg_cof": ("worker_22", 0.01732982882868463) }, "master": { "mean": ("master", 80.21548821548822), "max": ("master", 134), "reg_cof": ("master", -0.19400746078598444) } } } }, "stable_phase": { "wazuh-clusterd": { "USS(KB)": { "workers": { "mean": ("worker_12", 136051.86324786325), "max": ("worker_14", 183948.0), "reg_cof": ("worker_14", 689.7257023353081) }, "master": { "mean": ("master", 140449.8947368421), "max": ("master", 270984.0), "reg_cof": ("master", 288.0156528220629) } }, "CPU(%)": { "workers": { "mean": ("worker_9", 12.6), "max": ("worker_4", 34.8), "reg_cof": ("worker_15", 0.06064032391874895) }, "master": { "mean": ("master", 48.4587044534413), "max": ("master", 99.4), "reg_cof": ("master", 0.09772510089603965) } }, "FD": { "workers": { "mean": ("worker_9", 61.92727272727273), "max": ("worker_1", 64), "reg_cof": ("worker_15", 0.28031911840670387) }, "master": { "mean": ("master", 52.33198380566802), "max": ("master", 62), "reg_cof": ("master", 0.09891076872111303) } } } } }Important
The mentioned artifacts will not return the expected results due to the recent changes in the
csv_parser.pyfile applied in #4780. A previous version of the same should be used to replicate the behavior.We should review the methods involved in the statistics calculations to avoid wrong results and therefore failed performance tests in the future.
The text was updated successfully, but these errors were encountered: