time, nodes, nps

worker/games.py

@@ -392,13 +392,12 @@ def establish_validated_net(remote, testing_dir, net, global_cache):
             time.sleep(waitTime)
 
 
-def run_single_bench(engine, threads):
-    bench_sig = None
-    bench_nps = None
+def run_single_bench(engine, threads, depth):
+    bench_time, bench_nodes = None, None
 
     try:
         p = subprocess.Popen(
-            [engine, "bench", "16", str(threads), "13", "default", "depth"],
+            [engine, "bench", "16", str(threads), str(depth), "default", "depth"],
             stderr=subprocess.PIPE,
             stdout=subprocess.DEVNULL,
             universal_newlines=True,
@@ -407,59 +406,62 @@ def run_single_bench(engine, threads):
         )
 
         for line in iter(p.stderr.readline, ""):
+            if "Total time (ms)" in line:
+                bench_time = float(line.split(": ")[1].strip())
             if "Nodes searched" in line:
-                bench_sig = line.split(": ")[1].strip()
-            if "Nodes/second" in line:
-                bench_nps = float(line.split(": ")[1].strip())
+                bench_nodes = float(line.split(": ")[1].strip())
 
         p.wait()
     except (OSError, subprocess.SubprocessError) as e:
         raise e
 
-    return bench_sig, bench_nps
+    if bench_time is None or bench_nodes is None:
+        message = f"Unable to parse bench output of {engine.name}"
+        raise RunException(message)
 
+    return bench_time, bench_nodes
 
-def run_parallel_benches(engine, concurrency, threads, signature=None):
+
+def run_parallel_benches(engine, concurrency, threads, depth):
     with ProcessPoolExecutor(max_workers=concurrency) as executor:
         try:
             results = list(
                 executor.map(
-                    run_single_bench, [engine] * concurrency, [threads] * concurrency
+                    run_single_bench,
+                    [engine] * concurrency,
+                    [threads] * concurrency,
+                    [depth] * concurrency,
                 )
             )
         except Exception as e:
             message = f"Failed to run engine bench: {str(e)}"
             raise WorkerException(message, e=e)
 
-    bench_nps_values = []
-    for sig, nps in results:
-        if sig is None or nps is None:
-            message = f"Unable to parse bench output of {engine.name}"
-            raise RunException(message)
-
-        if threads == 1 and signature is not None and int(sig) != int(signature):
-            message = (
-                f"Wrong bench in {engine.name}, "
-                f"user expected: {signature} but worker got: {sig}"
-            )
-            raise RunException(message)
-
-        bench_nps_values.append(nps / threads)
+    bench_nodes_values = [bn for _, bn in results]
+    bench_time_values = [bt for bt, _ in results]
+    bench_nps_values = [1000 * bn / bt / threads for bt, bn in results]
 
-    total = sum(bench_nps_values)
-    average_nps = total / concurrency
+    mean_nodes = statistics.mean(bench_nodes_values)
+    bench_nodes = bench_nodes_values[0]
+    mean_time = statistics.mean(bench_time_values)
+    mean_nps = statistics.mean(bench_nps_values)
     min_nps = min(bench_nps_values)
     max_nps = max(bench_nps_values)
     median_nps = statistics.median(bench_nps_values)
-    std_nps = statistics.stdev(bench_nps_values) if len(bench_nps_values) > 1 else 0.0
+    std_nps = statistics.stdev(bench_nps_values) if len(bench_nps_values) > 1 else 0
 
     print(
-        f"Bench statistics for {engine.name}:\n"
-        f"average: {average_nps:.2f}, median: {median_nps:.2f}\n"
-        f"min: {min_nps:.2f}, max: {max_nps:.2f}, "
-        f"std: {std_nps:.2f}, std (%): {100 * std_nps / average_nps:.2f}"
+        f"Statistic at depth {depth} for {engine.name}:\n"
+        f"{'Mean nodes':<15}: {mean_nodes:15.2f}\n"
+        f"{'Mean time (ms)':<15}: {mean_time:15.2f}\n"
+        f"{'Mean nps':<15}: {mean_nps:15.2f}\n"
+        f"{'Median nps':<15}: {median_nps:15.2f}\n"
+        f"{'Min nps':<15}: {min_nps:15.2f}\n"
+        f"{'Max nps':<15}: {max_nps:15.2f}\n"
+        f"{'Std nps':<15}: {std_nps:15.2f}\n"
+        f"{'Std (%)':<15}: {100 * std_nps / mean_nps:15.2f}"
     )
-    return median_nps
+    return bench_nodes, mean_nps
 
 
 def verify_signature(engine, signature, games_concurrency, threads):
@@ -482,12 +484,17 @@ def verify_signature(engine, signature, games_concurrency, threads):
         )
         raise WorkerException(message)
 
-    # Run the benches with threads = 1 and validate the signature
-    bench_nps = run_parallel_benches(engine, games_concurrency, 1, signature)
+    # Run the benches with threads = 1 and depth = 13 to validate the signature
+    bench_nodes, _ = run_parallel_benches(engine, games_concurrency, 1, 13)
+    if int(bench_nodes) != int(signature):
+        message = (
+            f"Wrong bench in {engine.name}, "
+            f"user expected: {signature} but worker got: {bench_nodes}"
+        )
+        raise RunException(message)
 
-    # SMP test: run the benches with the required number of threads
-    if threads > 1:
-        bench_nps = run_parallel_benches(engine, games_concurrency, threads)
+    # Run the benches with the required number of threads and depth = 15
+    _, bench_nps = run_parallel_benches(engine, games_concurrency, threads, 15)
 
     return bench_nps, cpu_features
 

worker/sri.txt

@@ -1 +1 @@
-{"__version": 262, "updater.py": "dpMIHELIfaXCmCM4Bpge2G7Oikl1AiQq5tvQIQeS7WJcMSzVdVX/XvtRf5xFYRSQ", "worker.py": "FDGKksHE6Exf13q5xPPIop4zpjBfvlT6xOrXJwLy8oVmegRGqCMwUHcCzet/3Erg", "games.py": "1/oJ2pL0tac1uDsUjcDcKo9l97ZM+LX5EZXbeX4d+Xtq6QVgCKJInWobAdsSjudo"}
+{"__version": 262, "updater.py": "dpMIHELIfaXCmCM4Bpge2G7Oikl1AiQq5tvQIQeS7WJcMSzVdVX/XvtRf5xFYRSQ", "worker.py": "FDGKksHE6Exf13q5xPPIop4zpjBfvlT6xOrXJwLy8oVmegRGqCMwUHcCzet/3Erg", "games.py": "dbUBzVlAYQLl/5TOcPYnXcYrfcCf8NqoVCOTaonyEeBxKm4TkPdrb0dKORs8ubMu"}