diff --git a/benchmark-bot/src/render.ts b/benchmark-bot/src/render.ts
index 00bb695..fec56ee 100644
--- a/benchmark-bot/src/render.ts
+++ b/benchmark-bot/src/render.ts
@@ -134,9 +134,18 @@ function renderComparisonBlock(block: string, blockLimit: number): string {
return renderPreformatted(block, blockLimit);
}
- const summary = `${lines[0]}\n${lines[totalIndex]!.trimStart()}`;
+ const summaryIndexes = new Set([0, totalIndex]);
+ for (const [index, line] of lines.entries()) {
+ if (line.trimStart().startsWith("TASKS:")) {
+ summaryIndexes.add(index);
+ }
+ }
+ const summary = lines
+ .filter((_line, index) => summaryIndexes.has(index))
+ .map((line) => line.trimStart())
+ .join("\n");
const details = lines
- .filter((_line, index) => index !== 0 && index !== totalIndex)
+ .filter((_line, index) => !summaryIndexes.has(index))
.join("\n");
const summaryPrefix = "";
const summarySuffix = "
\n\n";
diff --git a/benchmark-bot/test/worker.test.ts b/benchmark-bot/test/worker.test.ts
index 5ab2876..69052f8 100644
--- a/benchmark-bot/test/worker.test.ts
+++ b/benchmark-bot/test/worker.test.ts
@@ -39,6 +39,7 @@ const TIMINGS = {
const COMPARISON = `=== Comparing tpch/sf1 results 'base' [prev] with 'head' [new] ===
q1: prev= 100 ms, new= 120 ms, diff=1.20 slower ✖
q2: prev= 200 ms, new= 150 ms, diff=1.33 faster ✔
+ TASKS: prev=20.0, new=18.0, diff=2.0 fewer (10.0%) (sum of per-query averages)
TOTAL: prev=300 ms, new=270 ms, diff=1.11 faster ✅`;
test("reports a completed comparison and consumes the job", async () => {
@@ -75,6 +76,8 @@ test("reports a completed comparison and consumes the job", async () => {
assert.match(comments[1]!, /Progress 3\/10/);
assert.match(comments[1]!, /Deploying the base revision/);
assert.match(comments[2]!, /TOTAL: prev=300 ms, new=270 ms/);
+ assert.match(comments[2]!, /TASKS: prev=20\.0, new=18\.0/);
+ assert.match(comments[2]!, /TASKS:.*TOTAL:.*<\/pre>\s*/s);
assert.match(comments[2]!, /Show full query output/);
assert.equal(comments[2]!.match(/=== Comparing/g)?.length, 1);
assert.equal(comments[2]!.match(/TOTAL:/g)?.length, 1);
diff --git a/benchmarks-remote/src/lib/results.ts b/benchmarks-remote/src/lib/results.ts
index 7153805..0acd7be 100644
--- a/benchmarks-remote/src/lib/results.ts
+++ b/benchmarks-remote/src/lib/results.ts
@@ -270,6 +270,8 @@ export class BenchmarkRun {
];
let totalTimePrev = 0;
let totalTimeNew = 0;
+ let totalTasksPrev = 0;
+ let totalTasksNew = 0;
const statsQErrorP50Prev: number[] = [];
const statsQErrorP50New: number[] = [];
const statsQErrorP95Prev: number[] = [];
@@ -284,6 +286,12 @@ export class BenchmarkRun {
if (timePrev !== undefined && timeNew !== undefined) {
totalTimePrev += timePrev;
totalTimeNew += timeNew;
+ const tasksPrev = prevQuery.averageTasks();
+ const tasksNew = query.averageTasks();
+ if (tasksPrev !== undefined && tasksNew !== undefined) {
+ totalTasksPrev += tasksPrev;
+ totalTasksNew += tasksNew;
+ }
statsQErrorP50Prev.push(
...prevQuery.iterations.flatMap((iteration) =>
iteration.statsQErrorP50 === undefined
@@ -342,6 +350,9 @@ export class BenchmarkRun {
);
if (qErrorP50) lines.push(qErrorP50);
if (qErrorP95) lines.push(qErrorP95);
+ lines.push(
+ `${taskComparison("TASKS", totalTasksPrev, totalTasksNew)} (sum of per-query averages)`,
+ );
lines.push(
`${"TOTAL".padStart(8)}: prev=${totalTimePrev.toString()} ms, new=${totalTimeNew.toString()} ms, diff=${factor.toFixed(2)} ${tag} ${emoji}`,
);
@@ -396,6 +407,16 @@ export class BenchResult {
return this.p50();
}
+ averageTasks(): number | undefined {
+ const values = this.iterations
+ .filter((iteration) => !iteration.error)
+ .map((iteration) => iteration.tasks);
+ if (values.length === 0) {
+ return undefined;
+ }
+ return values.reduce((sum, value) => sum + value, 0) / values.length;
+ }
+
comparison(prevQuery: BenchResult): string {
const prevError = prevQuery.iterations.find((value) => value.error)?.error;
const newError = this.iterations.find((value) => value.error)?.error;
@@ -412,6 +433,8 @@ export class BenchResult {
const p50Prev = prevQuery.p50();
const p50 = this.p50();
+ const tasksPrev = prevQuery.averageTasks();
+ const tasks = this.averageTasks();
let factor: number;
let tag: string;
@@ -426,7 +449,11 @@ export class BenchResult {
emoji = factor >= QUERY_HIGHLIGHT_THRESHOLD ? "❌" : "✖";
}
- return `${this.id.padStart(8)}: prev=${p50Prev.toString().padStart(4)} ms, new=${p50.toString().padStart(4)} ms, diff=${factor.toFixed(2)} ${tag} ${emoji}`;
+ const timeComparison = `${this.id.padStart(8)}: prev=${p50Prev.toString().padStart(4)} ms, new=${p50.toString().padStart(4)} ms, diff=${factor.toFixed(2)} ${tag} ${emoji}`;
+ if (tasksPrev === undefined || tasks === undefined) {
+ return timeComparison;
+ }
+ return `${timeComparison}, ${taskComparison("tasks", tasksPrev, tasks).trimStart()}`;
}
store(): void {
@@ -580,6 +607,24 @@ function qErrorComparison(
return undefined;
}
+function taskComparison(label: string, previous: number, next: number): string {
+ let difference: string;
+ if (next === previous) {
+ difference = "no change";
+ } else if (previous === 0) {
+ difference = `${formatTasks(next)} more`;
+ } else {
+ const absolute = Math.abs(next - previous);
+ const percentage = (absolute / previous) * 100;
+ difference = `${formatTasks(absolute)} ${next < previous ? "fewer" : "more"} (${percentage.toFixed(1)}%)`;
+ }
+ return `${label.padStart(8)}: prev=${formatTasks(previous)}, new=${formatTasks(next)}, diff=${difference}`;
+}
+
+function formatTasks(value: number): string {
+ return value.toFixed(1);
+}
+
function median(values: number[]): number | undefined {
if (values.length === 0) {
return undefined;
diff --git a/benchmarks-remote/sum-tasks.sh b/benchmarks-remote/sum-tasks.sh
new file mode 100755
index 0000000..a66d7d0
--- /dev/null
+++ b/benchmarks-remote/sum-tasks.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+result_dir=${1:?usage: sum-tasks.sh RESULT_DIRECTORY}
+
+if [[ ! -d ${result_dir} ]]; then
+ echo "Result directory does not exist: ${result_dir}" >&2
+ exit 1
+fi
+
+shopt -s nullglob
+result_files=("${result_dir}"/q*.json)
+if (( ${#result_files[@]} == 0 )); then
+ echo "No q*.json result files found in ${result_dir}" >&2
+ exit 1
+fi
+
+jq -s '
+ [.[].iterations[] | select(.error == null) | .tasks] | add // 0
+' "${result_files[@]}"
diff --git a/benchmarks-remote/test/compare-files.test.ts b/benchmarks-remote/test/compare-files.test.ts
index faf6b46..537ff98 100644
--- a/benchmarks-remote/test/compare-files.test.ts
+++ b/benchmarks-remote/test/compare-files.test.ts
@@ -13,9 +13,10 @@ function result(
resultName: string,
query: string,
elapsed: number,
+ tasks = 1,
): BenchResult {
const value = new BenchResult(dataset, resultName, query, root);
- value.iterations.push({ elapsed, plan: "", rowCount: 1, tasks: 1 });
+ value.iterations.push({ elapsed, plan: "", rowCount: 1, tasks });
return value;
}
@@ -47,12 +48,48 @@ test("builds comparisons from stored completed runs", () => {
const comparison = compareStoredResults("tpch/sf1", "base", "head", root);
assert.match(comparison, /^=== Comparing tpch\/sf1 results/);
assert.match(comparison, /q1: prev= 100 ms, new= 90 ms/);
+ assert.match(comparison, /tasks: prev=1\.0, new=1\.0, diff=no change/);
+ assert.match(comparison, /TASKS: prev=1\.0, new=1\.0, diff=no change/);
assert.match(comparison, /TOTAL: prev=100 ms, new=90 ms/);
} finally {
fs.rmSync(root, { recursive: true, force: true });
}
});
+test("compares per-query average tasks and sums them across queries", () => {
+ const root = fs.mkdtempSync(
+ path.join(os.tmpdir(), "benchmark-file-compare-"),
+ );
+ fs.mkdirSync(path.join(root, "tpch", "sf1"), { recursive: true });
+
+ try {
+ const baseQ1 = result(root, "tpch/sf1", "base", "q1", 100, 6);
+ baseQ1.iterations.push({ elapsed: 101, plan: "", rowCount: 1, tasks: 6 });
+ const headQ1 = result(root, "tpch/sf1", "head", "q1", 90, 4);
+ headQ1.iterations.push({ elapsed: 91, plan: "", rowCount: 1, tasks: 6 });
+ storeRun(root, "tpch/sf1", "base", [
+ baseQ1,
+ result(root, "tpch/sf1", "base", "q2", 200, 10),
+ ]);
+ storeRun(root, "tpch/sf1", "head", [
+ headQ1,
+ result(root, "tpch/sf1", "head", "q2", 180, 10),
+ ]);
+
+ const comparison = compareStoredResults("tpch/sf1", "base", "head", root);
+ assert.match(
+ comparison,
+ /q1:.*tasks: prev=6\.0, new=5\.0, diff=1\.0 fewer \(16\.7%\)/,
+ );
+ assert.match(
+ comparison,
+ /TASKS: prev=16\.0, new=15\.0, diff=1\.0 fewer \(6\.3%\) \(sum of per-query averages\)/,
+ );
+ } finally {
+ fs.rmSync(root, { recursive: true, force: true });
+ }
+});
+
test("keeps legacy per-query result directories comparable", () => {
const root = fs.mkdtempSync(
path.join(os.tmpdir(), "benchmark-file-compare-"),
diff --git a/benchmarks-remote/test/comparison-thresholds.test.ts b/benchmarks-remote/test/comparison-thresholds.test.ts
index 964ec63..afa6ac9 100644
--- a/benchmarks-remote/test/comparison-thresholds.test.ts
+++ b/benchmarks-remote/test/comparison-thresholds.test.ts
@@ -23,9 +23,9 @@ test("highlights individual queries at 1.5x", () => {
result("base", 150),
);
- assert.match(normalRegression, /1\.49 slower ✖$/);
- assert.match(highlightedRegression, /1\.50 slower ❌$/);
- assert.match(highlightedImprovement, /1\.50 faster ✅$/);
+ assert.match(normalRegression, /1\.49 slower ✖, tasks:/);
+ assert.match(highlightedRegression, /1\.50 slower ❌, tasks:/);
+ assert.match(highlightedImprovement, /1\.50 faster ✅, tasks:/);
});
test("highlights aggregate totals at 1.1x and prints TOTAL last", () => {
diff --git a/benchmarks-remote/test/results.test.ts b/benchmarks-remote/test/results.test.ts
index 73d5614..78a7c48 100644
--- a/benchmarks-remote/test/results.test.ts
+++ b/benchmarks-remote/test/results.test.ts
@@ -122,3 +122,27 @@ test("reports malformed previous-run manifests instead of treating them as absen
/Invalid previous run manifest/,
);
});
+
+test("averages task counts across successful measured iterations", () => {
+ const value = new BenchResult("tpch/sf1", "head", "q1");
+ value.iterations.push(
+ { elapsed: 10, plan: "", rowCount: 1, tasks: 4 },
+ { elapsed: 11, plan: "", rowCount: 1, tasks: 6 },
+ { elapsed: 0, plan: "", rowCount: 0, tasks: 0, error: "failed" },
+ );
+
+ assert.equal(value.averageTasks(), 5);
+});
+
+test("does not report an average task count without a successful iteration", () => {
+ const value = new BenchResult("tpch/sf1", "head", "q1");
+ value.iterations.push({
+ elapsed: 0,
+ plan: "",
+ rowCount: 0,
+ tasks: 0,
+ error: "failed",
+ });
+
+ assert.equal(value.averageTasks(), undefined);
+});