feat: show run mode in the UI (#947)

* Change quick mode parameters * Add BenchmarkRunModeEnum to ResultMetaInfo * Show BenchmarkRunModeEnum in the result screen * Show BenchmarkRunModeEnum in the history screen * Print error * Format Dart files * Remove obsolete sample generator and linter * Use dart run instead of flutter pub run * Show BenchmarkRunModeEnum in running screen * Fix UI issue with overflowing widgets * Update expected_throughput
mlcommons · Jan 21, 2025 · 390f130 · 390f130
1 parent 751e965
commit 390f130
Show file tree

Hide file tree

Showing 14 changed files with 91 additions and 163 deletions.
diff --git a/.github/workflows/format-and-lint.yml b/.github/workflows/format-and-lint.yml
@@ -90,8 +90,6 @@ jobs:
         run: make format/dart/pub-get
       - name: lint/dart
         run: make lint/dart
-      - name: result-schema
-        run: make lint/result-schema
 
   swift:
     needs: build-image

diff --git a/flutter/assets/tasks.pbtxt b/flutter/assets/tasks.pbtxt
@@ -15,7 +15,7 @@ task {
     }
     quick {
       min_query_count: 128
-      min_duration: 6
+      min_duration: 10
       max_duration: 60
     }
     rapid {
@@ -72,7 +72,7 @@ task {
     }
     quick {
       min_query_count: 128
-      min_duration: 6
+      min_duration: 10
       max_duration: 60
     }
     rapid {
@@ -129,7 +129,7 @@ task {
     }
     quick {
       min_query_count: 128
-      min_duration: 6
+      min_duration: 10
       max_duration: 60
     }
     rapid {
@@ -185,7 +185,7 @@ task {
     }
     quick {
       min_query_count: 128
-      min_duration: 6
+      min_duration: 10
       max_duration: 60
     }
     rapid {
@@ -238,7 +238,7 @@ task {
     }
     quick {
       min_query_count: 128
-      min_duration: 6
+      min_duration: 10
       max_duration: 60
     }
     rapid {
@@ -298,8 +298,8 @@ task {
     }
     rapid {
       min_query_count: 64
-      min_duration: 6
-      max_duration: 60
+      min_duration: 0
+      max_duration: 0
     }
   }
   datasets {
@@ -350,8 +350,8 @@ task {
     }
     quick {
       min_query_count: 128
-      min_duration: 6
-      max_duration: 30
+      min_duration: 10
+      max_duration: 40
     }
     rapid {
       min_query_count: 64

diff --git a/flutter/flutter.mk b/flutter/flutter.mk
@@ -29,7 +29,7 @@ endif
 endif
 
 flutter: flutter/prepare flutter/platform
-flutter/result: flutter/result/json flutter/result/gen-schema
+flutter/result: flutter/result/json
 flutter/prepare: flutter/pub flutter/result/json flutter/backend-list flutter/protobuf flutter/l10n flutter/build-info flutter/firebase-config flutter/set-windows-build-number
 flutter/check-release-env: flutter/check/official-build flutter/check/build-number
 flutter/test: flutter/test/unit flutter/test/integration
@@ -73,32 +73,11 @@ flutter/backend-list:
 		> flutter/lib/backend/list.gen.dart
 	dart format flutter/lib/backend/list.gen.dart
 
-RESULT_JSON_SAMPLE_PATH?=output/extended_result_example.json
-.PHONY: flutter/result/gen-extended-sample
-flutter/result/gen-extended-sample:
-	cd flutter && \
-		${_start_args} dart run \
-		--define=jsonFileName=../${RESULT_JSON_SAMPLE_PATH} \
-		lib/data/generation_helpers/write_json_sample.main.dart
-
-default_result_json_schema_path=tools/extended_result_schema.json
-RESULT_JSON_SCHEMA_PATH?=${default_result_json_schema_path}
-.PHONY: flutter/result/gen-schema
-flutter/result/gen-schema: flutter/result/gen-extended-sample
-	quicktype ${RESULT_JSON_SAMPLE_PATH} \
-		--lang schema \
-		--out ${RESULT_JSON_SCHEMA_PATH}
-	cd flutter && \
-		${_start_args} dart run \
-		--define=schemaPath=../${RESULT_JSON_SCHEMA_PATH} \
-		lib/data/generation_helpers/edit_json_schema.main.dart
-
 .PHONY: flutter/result/json
 flutter/result/json:
 	@echo "Generate .g.dart files for the @JsonSerializable annotation"
-	@# https://github.com/dart-lang/build/issues/2835#issuecomment-1047849076
-	cd flutter && ${_start_args} flutter packages pub get
-	cd flutter && ${_start_args} flutter --no-version-check pub run \
+	cd flutter && ${_start_args} dart pub get
+	cd flutter && ${_start_args} dart run \
 		build_runner build --delete-conflicting-outputs
 
 .PHONY: flutter/build-info

diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart
@@ -60,7 +60,7 @@ const Map<String, Map<String, Interval>> _imageClassificationV2 = {
     _kS24Ultra: Interval(min: 800, max: 1500),
   },
   _kMediatekBackend: {
-    _kDN2103: Interval(min: 5, max: 90),
+    _kDN2103: Interval(min: 4.5, max: 90),
     _kS10Plus: Interval(min: 400, max: 800)
   },
   _kSamsungBackend: {
@@ -88,11 +88,11 @@ const Map<String, Map<String, Interval>> _objectDetection = {
   },
   _kQtiBackend: {
     _kS22Ultra: Interval(min: 700, max: 1400),
-    _kS24Ultra: Interval(min: 1800, max: 2500),
+    _kS24Ultra: Interval(min: 1800, max: 2700),
   },
   _kMediatekBackend: {
     _kDN2103: Interval(min: 120, max: 210),
-    _kS10Plus: Interval(min: 1200, max: 1800)
+    _kS10Plus: Interval(min: 1200, max: 2000)
   },
   _kSamsungBackend: {
     _kS24: Interval(min: 1400, max: 2400),
@@ -123,7 +123,7 @@ const Map<String, Map<String, Interval>> _imageSegmentationV2 = {
   },
   _kMediatekBackend: {
     _kDN2103: Interval(min: 45, max: 70),
-    _kS10Plus: Interval(min: 800, max: 1400)
+    _kS10Plus: Interval(min: 800, max: 1500)
   },
   _kSamsungBackend: {
     _kS24: Interval(min: 800, max: 1500),
@@ -150,7 +150,7 @@ const Map<String, Map<String, Interval>> _naturalLanguageProcessing = {
   },
   _kQtiBackend: {
     _kS22Ultra: Interval(min: 100, max: 200),
-    _kS24Ultra: Interval(min: 250, max: 450),
+    _kS24Ultra: Interval(min: 250, max: 460),
   },
   _kMediatekBackend: {
     _kDN2103: Interval(min: 1, max: 6),
@@ -185,7 +185,7 @@ const Map<String, Map<String, Interval>> _superResolution = {
   },
   _kMediatekBackend: {
     _kDN2103: Interval(min: 5, max: 15),
-    _kS10Plus: Interval(min: 150, max: 280)
+    _kS10Plus: Interval(min: 150, max: 300)
   },
   _kSamsungBackend: {
     _kS24: Interval(min: 90, max: 180),
@@ -244,10 +244,10 @@ const Map<String, Map<String, Interval>> _imageClassificationOfflineV2 = {
   },
   _kQtiBackend: {
     _kS22Ultra: Interval(min: 250, max: 450),
-    _kS24Ultra: Interval(min: 900, max: 1600),
+    _kS24Ultra: Interval(min: 900, max: 1700),
   },
   _kMediatekBackend: {
-    _kDN2103: Interval(min: 5, max: 90),
+    _kDN2103: Interval(min: 4.5, max: 90),
     _kS10Plus: Interval(min: 700, max: 1200)
   },
   _kSamsungBackend: {

diff --git a/flutter/lib/benchmark/state.dart b/flutter/lib/benchmark/state.dart
@@ -106,7 +106,7 @@ class BenchmarkState extends ChangeNotifier {
       await setTaskConfig(name: _store.chosenConfigurationName);
       deferredLoadResources();
     } catch (e, trace) {
-      print("can't load resources: $e");
+      print("Can't load resources: $e");
       print(trace);
       error = e;
       stackTrace = trace;
@@ -122,7 +122,7 @@ class BenchmarkState extends ChangeNotifier {
     try {
       await loadResources(downloadMissing: false);
     } catch (e, trace) {
-      print("can't load resources: $e");
+      print("Can't load resources: $e");
       print(trace);
       error = e;
       stackTrace = trace;
@@ -164,7 +164,7 @@ class BenchmarkState extends ChangeNotifier {
       await state.setTaskConfig(name: store.chosenConfigurationName);
       state.deferredLoadResources();
     } catch (e, trace) {
-      print("can't load resources: $e");
+      print("Can't load resources: $e");
       print(trace);
       state.error = e;
       state.stackTrace = trace;
@@ -199,7 +199,7 @@ class BenchmarkState extends ChangeNotifier {
           taskSelection[kv.key] = kv.value as bool;
         }
       } catch (e, t) {
-        print('task selection parse fail: $e');
+        print('Task selection parse fail: $e');
         print(t);
       }
     }
@@ -255,7 +255,7 @@ class BenchmarkState extends ChangeNotifier {
           await taskRunner.runBenchmarks(_benchmarkStore, currentLogDir);
 
       if (lastResult == null) {
-        print('benchmark aborted');
+        print('Benchmark aborted');
       } else {
         print('Benchmarks finished');
 
@@ -267,6 +267,7 @@ class BenchmarkState extends ChangeNotifier {
       _doneRunning = taskRunner.aborting ? null : true;
     } catch (e) {
       _doneRunning = null;
+      print('Error: $e');
       rethrow;
     } finally {
       if (currentLogDir.isNotEmpty && !_store.keepLogs) {

diff --git a/flutter/lib/data/generation_helpers/edit_json_schema.main.dart b/flutter/lib/data/generation_helpers/edit_json_schema.main.dart
diff --git a/flutter/lib/data/generation_helpers/sample_generator.dart b/flutter/lib/data/generation_helpers/sample_generator.dart
@@ -1,6 +1,7 @@
 import 'package:uuid/uuid.dart';
 
 import 'package:mlperfbench/backend/loadgen_info.dart';
+import 'package:mlperfbench/benchmark/run_mode.dart';
 import 'package:mlperfbench/data/build_info/build_info.dart';
 import 'package:mlperfbench/data/environment/env_android.dart';
 import 'package:mlperfbench/data/environment/env_ios.dart';
@@ -132,6 +133,7 @@ class SampleGenerator {
           creationDate: DateTime.now(),
           uploadDate: DateTime.now(),
           uuid: const Uuid().v4(),
+          runMode: BenchmarkRunModeEnum.submissionRun,
         ),
         buildInfo: buildInfo,
         environmentInfo: envInfo,

diff --git a/flutter/lib/data/generation_helpers/write_json_sample.main.dart b/flutter/lib/data/generation_helpers/write_json_sample.main.dart
diff --git a/flutter/lib/data/meta_info.dart b/flutter/lib/data/meta_info.dart
@@ -1,16 +1,20 @@
 import 'package:json_annotation/json_annotation.dart';
 
+import 'package:mlperfbench/benchmark/run_mode.dart';
+
 part 'meta_info.g.dart';
 
 @JsonSerializable(fieldRename: FieldRename.snake)
 class ResultMetaInfo {
   final String uuid;
   final DateTime creationDate;
   final DateTime? uploadDate;
+  final BenchmarkRunModeEnum? runMode;
 
   ResultMetaInfo({
     required this.uuid,
     required this.creationDate,
+    required this.runMode,
     DateTime? uploadDate,
   }) : uploadDate = uploadDate?.toUtc();
 

diff --git a/flutter/lib/state/task_runner.dart b/flutter/lib/state/task_runner.dart
@@ -151,8 +151,13 @@ class TaskRunner {
     }
 
     final creationDate = DateTime.now();
+    final meta = ResultMetaInfo(
+      creationDate: creationDate,
+      uuid: const Uuid().v4(),
+      runMode: store.selectedBenchmarkRunMode,
+    );
     return ExtendedResult(
-      meta: ResultMetaInfo(creationDate: creationDate, uuid: const Uuid().v4()),
+      meta: meta,
       environmentInfo: DeviceInfo.instance.envInfo,
       results: exportResults,
       buildInfo: BuildInfoHelper.info,