Results for Token throughput based benchmarks

flutter/lib/backend/loadgen_info.dart

@@ -8,20 +8,28 @@
 @JsonSerializable(fieldRename: FieldRename.snake)
 class LoadgenInfo {
   final int queryCount;
-  final double latencyMean; // Mean latency in seconds
-  final double latency90; // 90th percentile in seconds
+  final double latencyMean; // Mean latency for a query in seconds
+  final double latency90; // 90th percentile for a query in seconds
+  final double latencyFirstTokenMean; // Mean TTFT latency in seconds
+  final double latencyFirstToken90; // 90th percentile TTFT in seconds
+  final double tokenThroughput; // A.K.A. TPS
   final bool isMinDurationMet;
   final bool isMinQueryMet;
   final bool isEarlyStoppingMet;
+  final bool isTokenBased;
   final bool isResultValid;
 
   LoadgenInfo({
     required this.queryCount,
     required this.latencyMean,
     required this.latency90,
+    required this.latencyFirstTokenMean,
+    required this.latencyFirstToken90,
+    required this.tokenThroughput,
     required this.isMinDurationMet,
     required this.isMinQueryMet,
     required this.isEarlyStoppingMet,
+    required this.isTokenBased,
     required this.isResultValid,
   });
 
@@ -58,29 +66,40 @@
   static Future<LoadgenInfo?> extractLoadgenInfo({
     required Stream<String> logLines,
   }) async {
+    // TODO possibly update these links
     // https://github.com/mlcommons/inference/blob/318cb131c0adf3bffcbc3379a502f40891331c54/loadgen/loadgen.cc#L1119
     const latencyKey = 'result_mean_latency_ns';
+    const latencyFirstTokenKey = 'result_first_token_mean_latency_ns';
     // https://github.com/mlcommons/inference/blob/318cb131c0adf3bffcbc3379a502f40891331c54/loadgen/loadgen.cc#L1055
     const queryCountKey = 'result_query_count';
     // https://github.com/mlcommons/inference/blob/318cb131c0adf3bffcbc3379a502f40891331c54/loadgen/loadgen.cc#L1121-L1124
     const latency90Key = 'result_90.00_percentile_latency_ns';
+    const latency90FirstTokenKey =
+        'result_first_token_90.00_percentile_latency_ns';
     // https://github.com/mlcommons/inference/blob/318cb131c0adf3bffcbc3379a502f40891331c54/loadgen/loadgen.cc#L1028-L1029
     const validityKey = 'result_validity';
     // https://github.com/mlcommons/inference/blob/318cb131c0adf3bffcbc3379a502f40891331c54/loadgen/loadgen.cc#L1033C23-L1035
     const minDurationMetKey = 'result_min_duration_met';
     const minQueriesMetKey = 'result_min_queries_met';
     const earlyStoppingMetKey = 'early_stopping_met';
+    const useTokenLatenciesKey = 'requested_use_token_latencies';
+    const tokenThroughputKey =
+        'result_token_throughput_with_loadgen_overhead'; // For some reason result_token_throughput is unreasonably low
 
     final result = await extractKeys(
       logLines: logLines,
       requiredKeys: {
         latencyKey,
+        latencyFirstTokenKey,
         queryCountKey,
         latency90Key,
+        latency90FirstTokenKey,
         validityKey,
         minDurationMetKey,
         minQueriesMetKey,
         earlyStoppingMetKey,
+        useTokenLatenciesKey,
+        tokenThroughputKey,
       },
     );
 
@@ -91,14 +110,26 @@
     final isResultValid = result[validityKey] as String == 'VALID';
 
     const nanosecondsPerSecond = 1000 * Duration.microsecondsPerSecond;
+    bool usesTokens = (result[useTokenLatenciesKey] ?? false) as bool;
 
     return LoadgenInfo(
       queryCount: result[queryCountKey] as int,
-      latencyMean: (result[latencyKey] as int) / nanosecondsPerSecond,
-      latency90: (result[latency90Key] as int) / nanosecondsPerSecond,
+      latencyMean:
+          !usesTokens ? (result[latencyKey] as int) / nanosecondsPerSecond : 0,
+      latencyFirstTokenMean: usesTokens
+          ? (result[latencyFirstTokenKey] as int) / nanosecondsPerSecond
+          : 0,
+      latency90: !usesTokens
+          ? (result[latency90Key] as int) / nanosecondsPerSecond
+          : 0,
+      latencyFirstToken90: usesTokens
+          ? (result[latency90FirstTokenKey] as int) / nanosecondsPerSecond
+          : 0,
+      tokenThroughput: usesTokens ? result[tokenThroughputKey] as double : 0,
       isMinDurationMet: result[minDurationMetKey] as bool,
       isMinQueryMet: result[minQueriesMetKey] as bool,
       isEarlyStoppingMet: result[earlyStoppingMetKey] as bool,
+      isTokenBased: usesTokens,
       isResultValid: isResultValid,
     );
   }

flutter/lib/data/generation_helpers/sample_generator.dart

@@ -47,6 +47,10 @@ class SampleGenerator {
           isMinQueryMet: true,
           isEarlyStoppingMet: true,
           isResultValid: true,
+          latencyFirstTokenMean: 0.123,
+          latencyFirstToken90: 0.123,
+          tokenThroughput: 12.345,
+          isTokenBased: false,
         ),
       );
 

flutter/lib/l10n/app_en.arb

@@ -4,6 +4,10 @@
   "unknown": "Unknown",
   "na": "N/A",
 
+  "unitSecond": "sec",
+  "unitTPS": "T/sec",
+  "unitQPS": "Q/sec",
+
   "menuHome": "MLPerf Mobile",
   "menuHistory": "History",
   "menuSettings": "Settings",
@@ -42,7 +46,7 @@
   "progressWaiting": "Wait for current benchmark to finish!",
 
   "resultsTitleUnverified": "Unverified",
-  "resultsTitlePerformance": "Results (qps)",
+  "resultsTitlePerformance": "Results",
   "resultsTitleAccuracy": "Results",
   "resultsTabTitlePerformance": "Performance",
   "resultsTabTitleAccuracy": "Accuracy",
@@ -171,6 +175,7 @@
   "historyDetailsDate": "Date",
   "historyDetailsUUID": "UUID",
   "historyDetailsAvgQps": "Average throughput (QPS)",
+  "historyDetailsAvgTps": "Average throughput (TPS)",
   "historyDetailsAppVersion": "App version",
   "historyDetailsAppVersionTemplate": "<version> (build <build>) (<buildType>)",
   "historyDetailsModelName": "Device model name",
@@ -193,6 +198,7 @@
   "historyRunDetailsBatchSize": "Batch size",
   "historyRunDetailsPerfTitle": "Performance run",
   "historyRunDetailsPerfQps": "Throughput",
+  "historyRunDetailsPerfTTFT": "Time to first token",
   "historyRunDetailsValid": "Run is valid",
   "historyRunDetailsDuration": "Duration",
   "historyRunDetailsSamples": "Samples count",

flutter/lib/state/task_runner.dart

@@ -375,9 +375,13 @@ class _NativeRunHelper {
       return null;
     }
     if (benchmark.info.isOffline) {
-      return result.numSamples / loadgenInfo.latency90;
+      return loadgenInfo.isTokenBased
+          ? loadgenInfo.tokenThroughput
+          : result.numSamples / loadgenInfo.latency90;
     } else {
-      return 1.0 / loadgenInfo.latency90;
+      return loadgenInfo.isTokenBased
+          ? loadgenInfo.tokenThroughput
+          : 1.0 / loadgenInfo.latency90;
     }
   }
 

flutter/lib/ui/auto_size_text.dart

@@ -52,6 +52,7 @@
     this.textScaleFactor = 1.0,
     this.maxLines,
     this.semanticsLabel,
+    this.textHeightBehavior,
     this.circularPadding = 20,
   }) : textSpan = null;
 
@@ -77,6 +78,7 @@
     this.textScaleFactor = 1.0,
     this.maxLines,
     this.semanticsLabel,
+    this.textHeightBehavior,
     this.circularPadding = 20,
   }) : data = null;
 
@@ -229,6 +231,8 @@
   /// ```
   final String? semanticsLabel;
 
+  final TextHeightBehavior? textHeightBehavior;
+
   final double circularPadding;
 
   @override
@@ -409,6 +413,7 @@
       maxLines: maxLines,
       locale: widget.locale,
       strutStyle: widget.strutStyle,
+      textHeightBehavior: widget.textHeightBehavior,
     );
 
     textPainter.layout(maxWidth: constraints.maxWidth);
@@ -436,6 +441,7 @@
           textScaler: const TextScaler.linear(1),
           maxLines: maxLines,
           semanticsLabel: widget.semanticsLabel,
+          textHeightBehavior: widget.textHeightBehavior,
         ),
       );
     } else {
@@ -452,6 +458,7 @@
         textScaler: TextScaler.linear(fontSize / style.fontSize!),
         maxLines: maxLines,
         semanticsLabel: widget.semanticsLabel,
+        textHeightBehavior: widget.textHeightBehavior,
       );
     }
   }

flutter/lib/ui/history/benchmark_export_result_screen.dart

@@ -74,7 +74,10 @@ class _BenchmarkExportResultScreenState
   List<Widget> _makePerformanceInfo(BenchmarkRunResult perf) {
     return [
       helper.makeInfo(l10n.historyRunDetailsPerfQps,
-          perf.throughput?.toUIString() ?? l10n.resultsNotAvailable),
+          '${perf.throughput!.toUIString()} ${perf.loadgenInfo!.isTokenBased ? l10n.unitTPS : l10n.unitQPS}'),
+      if (perf.loadgenInfo?.isTokenBased ?? false)
+        helper.makeInfo(l10n.historyRunDetailsPerfTTFT,
+            '${perf.loadgenInfo!.latencyFirstTokenMean.toStringAsFixed(2)} ${l10n.unitSecond}'),
       helper.makeInfo(l10n.historyRunDetailsValid,
           (perf.loadgenInfo?.isResultValid).toString()),
       helper.makeInfo(l10n.historyRunDetailsDuration,

flutter/lib/ui/history/extended_result_screen.dart

@@ -159,16 +159,21 @@
     return ListView(children: _makeBody());
   }
 
-  double calculateAverageThroughput(List<BenchmarkExportResult> results) {
+  double calculateAverageThroughput(List<BenchmarkExportResult> results,
+      {bool tokenBased = false}) {
     var throughput = 0.0;
     var count = 0;
     for (var item in results) {
-      if (item.performanceRun == null) {
+      if (item.performanceRun == null ||
+          item.performanceRun!.loadgenInfo!.isTokenBased != tokenBased) {
         continue;
       }
       throughput += item.performanceRun!.throughput!.value;
       count++;
     }
+    if (count == 0) {
+      return 0.0;
+    }
     return throughput / count;
   }
 
@@ -181,6 +186,9 @@
 
     final averageThroughput =
         calculateAverageThroughput(res.results).toStringAsFixed(2);
+    final averageTokenThroughput =
+        calculateAverageThroughput(res.results, tokenBased: true)
+            .toStringAsFixed(2);
 
     final appVersionType =
         (res.buildInfo.gitDirtyFlag || res.buildInfo.devTestFlag)
@@ -202,6 +210,7 @@
       helper.makeInfo(l10n.historyDetailsDate, date),
       helper.makeInfo(l10n.historyDetailsUUID, res.meta.uuid),
       helper.makeInfo(l10n.historyDetailsAvgQps, averageThroughput),
+      helper.makeInfo(l10n.historyDetailsAvgTps, averageTokenThroughput),
       helper.makeInfo(l10n.historyDetailsAppVersion, appVersion),
       helper.makeInfo(l10n.historyDetailsBackendName, backendName),
       helper.makeInfo(l10n.historyDetailsModelName, modelDescription),
@@ -238,8 +247,9 @@
     return RowData(
       isHeader: false,
       name: runInfo.benchmarkName,
-      throughput: runInfo.performanceRun?.throughput?.toUIString() ??
-          l10n.resultsNotAvailable,
+      throughput: runInfo.performanceRun != null
+          ? '${runInfo.performanceRun!.throughput!.toUIString()} ${runInfo.performanceRun!.loadgenInfo!.isTokenBased ? l10n.unitTPS : l10n.unitQPS}'
+          : l10n.resultsNotAvailable,
       throughputValid:
           runInfo.performanceRun?.loadgenInfo?.isResultValid ?? false,
       accuracy:

flutter/lib/ui/home/benchmark_config_section.dart

@@ -7,9 +7,8 @@
 import 'package:mlperfbench/benchmark/state.dart';
 import 'package:mlperfbench/localizations/app_localizations.dart';
 import 'package:mlperfbench/ui/app_styles.dart';
-import 'package:mlperfbench/ui/auto_size_text.dart';
 import 'package:mlperfbench/ui/error_dialog.dart';
-import 'package:mlperfbench/ui/home/app_drawer.dart';
+import 'package:mlperfbench/ui/home/benchmark_info_button.dart';
 import 'package:mlperfbench/ui/nil.dart';
 
 class BenchmarkConfigSection extends StatelessWidget {
@@ -59,7 +58,7 @@
                     height: 40,
                     child: TextButton(
                       onPressed: () {
-                        _showBottomSheet(context, benchmark);
+                        showBenchInfoBottomSheet(context, benchmark);
                       },
                       style: ElevatedButton.styleFrom(
                         backgroundColor: Colors.white,
@@ -149,89 +148,6 @@
     );
   }
 
-  void _showBottomSheet(BuildContext context, Benchmark benchmark) {
-    final l10n = AppLocalizations.of(context)!;
-
-    final info = benchmark.info.getLocalizedInfo(l10n);
-
-    const double sidePadding = 18.0;
-    // 48pt original height + vertical padding of 18pt in each direction
-    const double headHeight = 48.0 + (18.0 * 2);
-    const double footHeight = 36.0;
-
-    showModalBottomSheet(
-      context: context,
-      isDismissible: false,
-      enableDrag: false,
-      isScrollControlled: true,
-      shape: const RoundedRectangleBorder(
-          borderRadius: BorderRadius.vertical(top: Radius.circular(24))),
-      builder: (context) => Padding(
-        padding: const EdgeInsets.symmetric(horizontal: sidePadding),
-        child: Column(
-          mainAxisSize: MainAxisSize.min,
-          children: [
-            SizedBox(
-              height: headHeight,
-              width: MediaQuery.of(context).size.width - sidePadding,
-              child: Center(
-                child: Row(
-                  mainAxisSize: MainAxisSize.max,
-                  mainAxisAlignment: MainAxisAlignment.spaceBetween,
-                  crossAxisAlignment: CrossAxisAlignment.center,
-                  children: [
-                    Expanded(
-                      child: AutoSizeText(
-                        benchmark.taskConfig.name,
-                        overflow: TextOverflow.ellipsis,
-                        maxLines: 1, //can be changed to 2 without issue
-                        textAlign: TextAlign.left,
-                        style: const TextStyle(
-                          fontWeight: FontWeight.bold,
-                          fontSize: 28,
-                        ),
-                      ),
-                    ),
-                    IconButton(
-                      splashRadius: 24,
-                      onPressed: () => Navigator.pop(context),
-                      icon: const Icon(Icons.close, color: Colors.grey),
-                    ),
-                  ],
-                ),
-              ),
-            ),
-            LayoutBuilder(builder: (context, constraints) {
-              print(constraints.maxHeight);
-              return ConstrainedBox(
-                constraints: constraints.copyWith(
-                    maxHeight: constraints.maxHeight != double.infinity
-                        ? constraints.maxHeight
-                        : MediaQuery.of(context).size.height - headHeight),
-                child: ScrollConfiguration(
-                  behavior: NoGlowScrollBehavior(),
-                  child: SingleChildScrollView(
-                    child: Column(
-                      children: [
-                        Text(
-                          info.detailsContent,
-                          style: const TextStyle(fontSize: 16),
-                        ),
-                        const SizedBox(
-                          height: footHeight,
-                        )
-                      ],
-                    ),
-                  ),
-                ),
-              );
-            }),
-          ],
-        ),
-      ),
-    );
-  }
-
   Widget _delegateChoice(
       Benchmark benchmark, BuildContext context, BenchmarkState state) {
     final selected = benchmark.benchmarkSettings.delegateSelected;