diff --git a/worker/games.py b/worker/games.py index dff2c8cb8..6595b1624 100644 --- a/worker/games.py +++ b/worker/games.py @@ -392,13 +392,12 @@ def establish_validated_net(remote, testing_dir, net, global_cache): time.sleep(waitTime) -def run_single_bench(engine, threads): - bench_sig = None - bench_nps = None +def run_single_bench(engine, threads, depth): + bench_time, bench_nodes = None, None try: p = subprocess.Popen( - [engine, "bench", "16", str(threads), "13", "default", "depth"], + [engine, "bench", "16", str(threads), str(depth), "default", "depth"], stderr=subprocess.PIPE, stdout=subprocess.DEVNULL, universal_newlines=True, @@ -407,59 +406,62 @@ def run_single_bench(engine, threads): ) for line in iter(p.stderr.readline, ""): + if "Total time (ms)" in line: + bench_time = float(line.split(": ")[1].strip()) if "Nodes searched" in line: - bench_sig = line.split(": ")[1].strip() - if "Nodes/second" in line: - bench_nps = float(line.split(": ")[1].strip()) + bench_nodes = float(line.split(": ")[1].strip()) p.wait() except (OSError, subprocess.SubprocessError) as e: raise e - return bench_sig, bench_nps + if bench_time is None or bench_nodes is None: + message = f"Unable to parse bench output of {engine.name}" + raise RunException(message) + return bench_time, bench_nodes -def run_parallel_benches(engine, concurrency, threads, signature=None): + +def run_parallel_benches(engine, concurrency, threads, depth): with ProcessPoolExecutor(max_workers=concurrency) as executor: try: results = list( executor.map( - run_single_bench, [engine] * concurrency, [threads] * concurrency + run_single_bench, + [engine] * concurrency, + [threads] * concurrency, + [depth] * concurrency, ) ) except Exception as e: message = f"Failed to run engine bench: {str(e)}" raise WorkerException(message, e=e) - bench_nps_values = [] - for sig, nps in results: - if sig is None or nps is None: - message = f"Unable to parse bench output of {engine.name}" - raise RunException(message) - - if threads == 1 and signature is not None and int(sig) != int(signature): - message = ( - f"Wrong bench in {engine.name}, " - f"user expected: {signature} but worker got: {sig}" - ) - raise RunException(message) - - bench_nps_values.append(nps / threads) + bench_nodes_values = [bn for _, bn in results] + bench_time_values = [bt for bt, _ in results] + bench_nps_values = [1000 * bn / bt / threads for bt, bn in results] - total = sum(bench_nps_values) - average_nps = total / concurrency + mean_nodes = statistics.mean(bench_nodes_values) + bench_nodes = bench_nodes_values[0] + mean_time = statistics.mean(bench_time_values) + mean_nps = statistics.mean(bench_nps_values) min_nps = min(bench_nps_values) max_nps = max(bench_nps_values) median_nps = statistics.median(bench_nps_values) - std_nps = statistics.stdev(bench_nps_values) if len(bench_nps_values) > 1 else 0.0 + std_nps = statistics.stdev(bench_nps_values) if len(bench_nps_values) > 1 else 0 print( - f"Bench statistics for {engine.name}:\n" - f"average: {average_nps:.2f}, median: {median_nps:.2f}\n" - f"min: {min_nps:.2f}, max: {max_nps:.2f}, " - f"std: {std_nps:.2f}, std (%): {100 * std_nps / average_nps:.2f}" + f"Statistic at depth {depth} for {engine.name}:\n" + f"{'Mean nodes':<15}: {mean_nodes:15.2f}\n" + f"{'Mean time (ms)':<15}: {mean_time:15.2f}\n" + f"{'Mean nps':<15}: {mean_nps:15.2f}\n" + f"{'Median nps':<15}: {median_nps:15.2f}\n" + f"{'Min nps':<15}: {min_nps:15.2f}\n" + f"{'Max nps':<15}: {max_nps:15.2f}\n" + f"{'Std nps':<15}: {std_nps:15.2f}\n" + f"{'Std (%)':<15}: {100 * std_nps / mean_nps:15.2f}" ) - return median_nps + return bench_nodes, mean_nps def verify_signature(engine, signature, games_concurrency, threads): @@ -482,12 +484,17 @@ def verify_signature(engine, signature, games_concurrency, threads): ) raise WorkerException(message) - # Run the benches with threads = 1 and validate the signature - bench_nps = run_parallel_benches(engine, games_concurrency, 1, signature) + # Run the benches with threads = 1 and depth = 13 to validate the signature + bench_nodes, _ = run_parallel_benches(engine, games_concurrency, 1, 13) + if int(bench_nodes) != int(signature): + message = ( + f"Wrong bench in {engine.name}, " + f"user expected: {signature} but worker got: {bench_nodes}" + ) + raise RunException(message) - # SMP test: run the benches with the required number of threads - if threads > 1: - bench_nps = run_parallel_benches(engine, games_concurrency, threads) + # Run the benches with the required number of threads and depth = 15 + _, bench_nps = run_parallel_benches(engine, games_concurrency, threads, 15) return bench_nps, cpu_features diff --git a/worker/sri.txt b/worker/sri.txt index 92f30fceb..547b3c21f 100644 --- a/worker/sri.txt +++ b/worker/sri.txt @@ -1 +1 @@ -{"__version": 262, "updater.py": "dpMIHELIfaXCmCM4Bpge2G7Oikl1AiQq5tvQIQeS7WJcMSzVdVX/XvtRf5xFYRSQ", "worker.py": "FDGKksHE6Exf13q5xPPIop4zpjBfvlT6xOrXJwLy8oVmegRGqCMwUHcCzet/3Erg", "games.py": "1/oJ2pL0tac1uDsUjcDcKo9l97ZM+LX5EZXbeX4d+Xtq6QVgCKJInWobAdsSjudo"} +{"__version": 262, "updater.py": "dpMIHELIfaXCmCM4Bpge2G7Oikl1AiQq5tvQIQeS7WJcMSzVdVX/XvtRf5xFYRSQ", "worker.py": "FDGKksHE6Exf13q5xPPIop4zpjBfvlT6xOrXJwLy8oVmegRGqCMwUHcCzet/3Erg", "games.py": "dbUBzVlAYQLl/5TOcPYnXcYrfcCf8NqoVCOTaonyEeBxKm4TkPdrb0dKORs8ubMu"}