Report benchmark task counts

benchmark-bot/src/render.ts

@@ -134,9 +134,18 @@ function renderComparisonBlock(block: string, blockLimit: number): string {
     return renderPreformatted(block, blockLimit);
   }
 
-  const summary = `${lines[0]}\n${lines[totalIndex]!.trimStart()}`;
+  const summaryIndexes = new Set([0, totalIndex]);
+  for (const [index, line] of lines.entries()) {
+    if (line.trimStart().startsWith("TASKS:")) {
+      summaryIndexes.add(index);
+    }
+  }
+  const summary = lines
+    .filter((_line, index) => summaryIndexes.has(index))
+    .map((line) => line.trimStart())
+    .join("\n");
   const details = lines
-    .filter((_line, index) => index !== 0 && index !== totalIndex)
+    .filter((_line, index) => !summaryIndexes.has(index))
     .join("\n");
   const summaryPrefix = "<pre>";
   const summarySuffix = "</pre>\n\n";

benchmark-bot/test/worker.test.ts

@@ -39,6 +39,7 @@ const TIMINGS = {
 const COMPARISON = `=== Comparing tpch/sf1 results 'base' [prev] with 'head' [new] ===
       q1: prev= 100 ms, new= 120 ms, diff=1.20 slower ✖
       q2: prev= 200 ms, new= 150 ms, diff=1.33 faster ✔
+   TASKS: prev=20.0, new=18.0, diff=2.0 fewer (10.0%) (sum of per-query averages)
    TOTAL: prev=300 ms, new=270 ms, diff=1.11 faster ✅`;
 
 test("reports a completed comparison and consumes the job", async () => {
@@ -75,6 +76,8 @@ test("reports a completed comparison and consumes the job", async () => {
     assert.match(comments[1]!, /Progress 3\/10/);
     assert.match(comments[1]!, /Deploying the base revision/);
     assert.match(comments[2]!, /TOTAL: prev=300 ms, new=270 ms/);
+    assert.match(comments[2]!, /TASKS: prev=20\.0, new=18\.0/);
+    assert.match(comments[2]!, /TASKS:.*TOTAL:.*<\/pre>\s*<details>/s);
     assert.match(comments[2]!, /Show full query output/);
     assert.equal(comments[2]!.match(/=== Comparing/g)?.length, 1);
     assert.equal(comments[2]!.match(/TOTAL:/g)?.length, 1);

benchmarks-remote/src/lib/results.ts

@@ -270,6 +270,8 @@ export class BenchmarkRun {
     ];
     let totalTimePrev = 0;
     let totalTimeNew = 0;
+    let totalTasksPrev = 0;
+    let totalTasksNew = 0;
     const statsQErrorP50Prev: number[] = [];
     const statsQErrorP50New: number[] = [];
     const statsQErrorP95Prev: number[] = [];
@@ -284,6 +286,12 @@ export class BenchmarkRun {
       if (timePrev !== undefined && timeNew !== undefined) {
         totalTimePrev += timePrev;
         totalTimeNew += timeNew;
+        const tasksPrev = prevQuery.averageTasks();
+        const tasksNew = query.averageTasks();
+        if (tasksPrev !== undefined && tasksNew !== undefined) {
+          totalTasksPrev += tasksPrev;
+          totalTasksNew += tasksNew;
+        }
         statsQErrorP50Prev.push(
           ...prevQuery.iterations.flatMap((iteration) =>
             iteration.statsQErrorP50 === undefined
@@ -342,6 +350,9 @@ export class BenchmarkRun {
     );
     if (qErrorP50) lines.push(qErrorP50);
     if (qErrorP95) lines.push(qErrorP95);
+    lines.push(
+      `${taskComparison("TASKS", totalTasksPrev, totalTasksNew)} (sum of per-query averages)`,
+    );
     lines.push(
       `${"TOTAL".padStart(8)}: prev=${totalTimePrev.toString()} ms, new=${totalTimeNew.toString()} ms, diff=${factor.toFixed(2)} ${tag} ${emoji}`,
     );
@@ -396,6 +407,16 @@ export class BenchResult {
     return this.p50();
   }
 
+  averageTasks(): number | undefined {
+    const values = this.iterations
+      .filter((iteration) => !iteration.error)
+      .map((iteration) => iteration.tasks);
+    if (values.length === 0) {
+      return undefined;
+    }
+    return values.reduce((sum, value) => sum + value, 0) / values.length;
+  }
+
   comparison(prevQuery: BenchResult): string {
     const prevError = prevQuery.iterations.find((value) => value.error)?.error;
     const newError = this.iterations.find((value) => value.error)?.error;
@@ -412,6 +433,8 @@ export class BenchResult {
 
     const p50Prev = prevQuery.p50();
     const p50 = this.p50();
+    const tasksPrev = prevQuery.averageTasks();
+    const tasks = this.averageTasks();
 
     let factor: number;
     let tag: string;
@@ -426,7 +449,11 @@ export class BenchResult {
       emoji = factor >= QUERY_HIGHLIGHT_THRESHOLD ? "❌" : "✖";
     }
 
-    return `${this.id.padStart(8)}: prev=${p50Prev.toString().padStart(4)} ms, new=${p50.toString().padStart(4)} ms, diff=${factor.toFixed(2)} ${tag} ${emoji}`;
+    const timeComparison = `${this.id.padStart(8)}: prev=${p50Prev.toString().padStart(4)} ms, new=${p50.toString().padStart(4)} ms, diff=${factor.toFixed(2)} ${tag} ${emoji}`;
+    if (tasksPrev === undefined || tasks === undefined) {
+      return timeComparison;
+    }
+    return `${timeComparison}, ${taskComparison("tasks", tasksPrev, tasks).trimStart()}`;
   }
 
   store(): void {
@@ -580,6 +607,24 @@ function qErrorComparison(
   return undefined;
 }
 
+function taskComparison(label: string, previous: number, next: number): string {
+  let difference: string;
+  if (next === previous) {
+    difference = "no change";
+  } else if (previous === 0) {
+    difference = `${formatTasks(next)} more`;
+  } else {
+    const absolute = Math.abs(next - previous);
+    const percentage = (absolute / previous) * 100;
+    difference = `${formatTasks(absolute)} ${next < previous ? "fewer" : "more"} (${percentage.toFixed(1)}%)`;
+  }
+  return `${label.padStart(8)}: prev=${formatTasks(previous)}, new=${formatTasks(next)}, diff=${difference}`;
+}
+
+function formatTasks(value: number): string {
+  return value.toFixed(1);
+}
+
 function median(values: number[]): number | undefined {
   if (values.length === 0) {
     return undefined;

benchmarks-remote/sum-tasks.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+result_dir=${1:?usage: sum-tasks.sh RESULT_DIRECTORY}
+
+if [[ ! -d ${result_dir} ]]; then
+  echo "Result directory does not exist: ${result_dir}" >&2
+  exit 1
+fi
+
+shopt -s nullglob
+result_files=("${result_dir}"/q*.json)
+if (( ${#result_files[@]} == 0 )); then
+  echo "No q*.json result files found in ${result_dir}" >&2
+  exit 1
+fi
+
+jq -s '
+  [.[].iterations[] | select(.error == null) | .tasks] | add // 0
+' "${result_files[@]}"

benchmarks-remote/test/compare-files.test.ts

@@ -13,9 +13,10 @@ function result(
   resultName: string,
   query: string,
   elapsed: number,
+  tasks = 1,
 ): BenchResult {
   const value = new BenchResult(dataset, resultName, query, root);
-  value.iterations.push({ elapsed, plan: "", rowCount: 1, tasks: 1 });
+  value.iterations.push({ elapsed, plan: "", rowCount: 1, tasks });
   return value;
 }
 
@@ -47,12 +48,48 @@ test("builds comparisons from stored completed runs", () => {
     const comparison = compareStoredResults("tpch/sf1", "base", "head", root);
     assert.match(comparison, /^=== Comparing tpch\/sf1 results/);
     assert.match(comparison, /q1: prev= 100 ms, new=  90 ms/);
+    assert.match(comparison, /tasks: prev=1\.0, new=1\.0, diff=no change/);
+    assert.match(comparison, /TASKS: prev=1\.0, new=1\.0, diff=no change/);
     assert.match(comparison, /TOTAL: prev=100 ms, new=90 ms/);
   } finally {
     fs.rmSync(root, { recursive: true, force: true });
   }
 });
 
+test("compares per-query average tasks and sums them across queries", () => {
+  const root = fs.mkdtempSync(
+    path.join(os.tmpdir(), "benchmark-file-compare-"),
+  );
+  fs.mkdirSync(path.join(root, "tpch", "sf1"), { recursive: true });
+
+  try {
+    const baseQ1 = result(root, "tpch/sf1", "base", "q1", 100, 6);
+    baseQ1.iterations.push({ elapsed: 101, plan: "", rowCount: 1, tasks: 6 });
+    const headQ1 = result(root, "tpch/sf1", "head", "q1", 90, 4);
+    headQ1.iterations.push({ elapsed: 91, plan: "", rowCount: 1, tasks: 6 });
+    storeRun(root, "tpch/sf1", "base", [
+      baseQ1,
+      result(root, "tpch/sf1", "base", "q2", 200, 10),
+    ]);
+    storeRun(root, "tpch/sf1", "head", [
+      headQ1,
+      result(root, "tpch/sf1", "head", "q2", 180, 10),
+    ]);
+
+    const comparison = compareStoredResults("tpch/sf1", "base", "head", root);
+    assert.match(
+      comparison,
+      /q1:.*tasks: prev=6\.0, new=5\.0, diff=1\.0 fewer \(16\.7%\)/,
+    );
+    assert.match(
+      comparison,
+      /TASKS: prev=16\.0, new=15\.0, diff=1\.0 fewer \(6\.3%\) \(sum of per-query averages\)/,
+    );
+  } finally {
+    fs.rmSync(root, { recursive: true, force: true });
+  }
+});
+
 test("keeps legacy per-query result directories comparable", () => {
   const root = fs.mkdtempSync(
     path.join(os.tmpdir(), "benchmark-file-compare-"),

benchmarks-remote/test/comparison-thresholds.test.ts

@@ -23,9 +23,9 @@ test("highlights individual queries at 1.5x", () => {
     result("base", 150),
   );
 
-  assert.match(normalRegression, /1\.49 slower ✖$/);
-  assert.match(highlightedRegression, /1\.50 slower ❌$/);
-  assert.match(highlightedImprovement, /1\.50 faster ✅$/);
+  assert.match(normalRegression, /1\.49 slower ✖, tasks:/);
+  assert.match(highlightedRegression, /1\.50 slower ❌, tasks:/);
+  assert.match(highlightedImprovement, /1\.50 faster ✅, tasks:/);
 });
 
 test("highlights aggregate totals at 1.1x and prints TOTAL last", () => {

benchmarks-remote/test/results.test.ts

@@ -122,3 +122,27 @@ test("reports malformed previous-run manifests instead of treating them as absen
     /Invalid previous run manifest/,
   );
 });
+
+test("averages task counts across successful measured iterations", () => {
+  const value = new BenchResult("tpch/sf1", "head", "q1");
+  value.iterations.push(
+    { elapsed: 10, plan: "", rowCount: 1, tasks: 4 },
+    { elapsed: 11, plan: "", rowCount: 1, tasks: 6 },
+    { elapsed: 0, plan: "", rowCount: 0, tasks: 0, error: "failed" },
+  );
+
+  assert.equal(value.averageTasks(), 5);
+});
+
+test("does not report an average task count without a successful iteration", () => {
+  const value = new BenchResult("tpch/sf1", "head", "q1");
+  value.iterations.push({
+    elapsed: 0,
+    plan: "",
+    rowCount: 0,
+    tasks: 0,
+    error: "failed",
+  });
+
+  assert.equal(value.averageTasks(), undefined);
+});