feat: Compare sample_entity_whole_population relative to "sentinel" CPU-bound benchmark

ixa-bench/criterion/sample_entity_scaling.rs

@@ -5,6 +5,7 @@ use std::time::Instant;
 
 use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
 use ixa::prelude::*;
+use ixa::rand::Rng;
 
 define_rng!(SampleScalingRng);
 
@@ -16,6 +17,9 @@ define_multi_property!((Species, Region), Mosquito);
 define_property!(struct Unindexed10(u8), Mosquito);
 
 const POPULATION_SIZES: [usize; 3] = [1_000, 10_000, 100_000];
+const WHOLE_POPULATION_SENTINEL: &str = "sentinel";
+const WHOLE_POPULATION_SENTINEL_UPPER: u32 = 100_000;
+const WHOLE_POPULATION_SENTINEL_RESULTS_SIZE: usize = 0;
 
 // Shared place to stash "ns per sample" per (bench_name, size)
 type Results = Arc<Mutex<BTreeMap<(String, usize), f64>>>;
@@ -69,6 +73,28 @@ fn print_scaling_summary(results: &BTreeMap<(String, usize), f64>, bench_name: &
     }
 }
 
+fn print_whole_population_sentinel_summary(
+    results: &BTreeMap<(String, usize), f64>,
+    bench_name: &str,
+) {
+    let sentinel_name = format!("{bench_name}/{WHOLE_POPULATION_SENTINEL}");
+    let Some(sentinel_ns) = results.get(&(sentinel_name, WHOLE_POPULATION_SENTINEL_RESULTS_SIZE))
+    else {
+        return;
+    };
+
+    eprintln!("\n=== Sentinel-relative summary: {bench_name} ===");
+    for &size in &POPULATION_SIZES {
+        let Some(sample_ns) = results.get(&(bench_name.to_string(), size)) else {
+            continue;
+        };
+        eprintln!(
+            "  n={size:>7}: sample={sample_ns:.2} ns/sample, sentinel={sentinel_ns:.2} ns/sample, ratio={:.3}x",
+            sample_ns / sentinel_ns,
+        );
+    }
+}
+
 // This is so we can do an O_n analysis of the benchmark
 fn bench_ns_per_sample<F>(bencher: &mut criterion::Bencher, mut f: F) -> f64
 where
@@ -102,8 +128,9 @@ pub fn bench_sample_entity_whole_population(c: &mut Criterion, results: Results)
 
         group.bench_with_input(BenchmarkId::from_parameter(size), &size, |b, _| {
             let ns = bench_ns_per_sample(b, || {
-                let _: Option<EntityId<Mosquito>> =
+                let sampled: Option<EntityId<Mosquito>> =
                     context.sample_entity(SampleScalingRng, Mosquito);
+                black_box(sampled);
             });
 
             results
@@ -113,6 +140,24 @@ pub fn bench_sample_entity_whole_population(c: &mut Criterion, results: Results)
         });
     }
 
+    let sentinel_context = setup_context(WHOLE_POPULATION_SENTINEL_UPPER as usize);
+    group.bench_function(WHOLE_POPULATION_SENTINEL, |b| {
+        let ns = bench_ns_per_sample(b, || {
+            let sampled = sentinel_context.sample(SampleScalingRng, |rng| {
+                rng.random_range(0..WHOLE_POPULATION_SENTINEL_UPPER)
+            });
+            black_box(sampled);
+        });
+
+        results.lock().unwrap().insert(
+            (
+                format!("{bench_name}/{WHOLE_POPULATION_SENTINEL}"),
+                WHOLE_POPULATION_SENTINEL_RESULTS_SIZE,
+            ),
+            ns,
+        );
+    });
+
     group.finish();
 }
 
@@ -205,6 +250,7 @@ fn sample_entity_scaling(c: &mut Criterion) {
     print_scaling_summary(&results, "sample_entity_single_property_indexed");
     print_scaling_summary(&results, "sample_entity_multi_property_indexed");
     print_scaling_summary(&results, "sample_entity_single_property_unindexed");
+    print_whole_population_sentinel_summary(&results, "sample_entity_whole_population");
 }
 
 criterion_group!(benches, sample_entity_scaling);

scripts/bench_results.mjs

@@ -25,6 +25,9 @@ import { fileURLToPath } from 'node:url';
 const __filename = fileURLToPath(import.meta.url);
 
 const DEFAULT_MAX_INPUT_BYTES = 100 * 1024 * 1024; // 100 MiB
+const SAMPLE_ENTITY_WHOLE_POPULATION_PREFIX = 'sample_entity_whole_population/';
+const SAMPLE_ENTITY_WHOLE_POPULATION_SENTINEL = 'sample_entity_whole_population/sentinel';
+const SAMPLE_ENTITY_WHOLE_POPULATION_SIZES = ['1000', '10000', '100000'];
 
 function maxInputBytes() {
   const raw = process.env.BENCH_RESULTS_MAX_INPUT_BYTES;
@@ -150,6 +153,48 @@ function parseCriterionCompareLog(text) {
   return Array.from(resultsByName.values());
 }
 
+function hasFinitePositiveTriple(values) {
+  return (
+    Array.isArray(values) &&
+    values.length >= 3 &&
+    values.slice(0, 3).every((value) => Number.isFinite(value) && value > 0)
+  );
+}
+
+function ratioText(value) {
+  return `${value.toFixed(3)}x`;
+}
+
+function addSampleEntityWholePopulationSentinelRatios(criterionTimings) {
+  const byName = new Map(criterionTimings.map((entry) => [entry.name, entry]));
+  const sentinel = byName.get(SAMPLE_ENTITY_WHOLE_POPULATION_SENTINEL);
+  if (!hasFinitePositiveTriple(sentinel?.time_sec)) return criterionTimings;
+
+  for (const size of SAMPLE_ENTITY_WHOLE_POPULATION_SIZES) {
+    const sampleName = `${SAMPLE_ENTITY_WHOLE_POPULATION_PREFIX}${size}`;
+    const sample = byName.get(sampleName);
+    if (!hasFinitePositiveTriple(sample?.time_sec)) continue;
+
+    const ratio = [
+      sample.time_sec[0] / sentinel.time_sec[2],
+      sample.time_sec[1] / sentinel.time_sec[1],
+      sample.time_sec[2] / sentinel.time_sec[0],
+    ];
+    if (!ratio.every((value) => Number.isFinite(value) && value > 0)) continue;
+
+    sample.relative_to_sentinel = {
+      kind: 'sentinel_ratio',
+      sentinel: SAMPLE_ENTITY_WHOLE_POPULATION_SENTINEL,
+      ratio,
+      ratio_text: ratio.map(ratioText),
+      method:
+        'sample_time / sentinel_time; lower=sample_lower/sentinel_upper; estimate=sample_estimate/sentinel_estimate; upper=sample_upper/sentinel_lower',
+    };
+  }
+
+  return criterionTimings;
+}
+
 function parseHyperfineJson(hyperfineJson) {
   if (!hyperfineJson || !Array.isArray(hyperfineJson.results)) return [];
 
@@ -224,7 +269,9 @@ function main() {
   const hyperfineTimings = parseHyperfineJson(hyperfineJson);
 
   const criterionCompareLog = readTextIfExists(criterionLogPath);
-  const criterionTimings = parseCriterionCompareLog(criterionCompareLog);
+  const criterionTimings = addSampleEntityWholePopulationSentinelRatios(
+    parseCriterionCompareLog(criterionCompareLog)
+  );
 
   const payload = {
     schema: 1,

scripts/format_bench_pr_comment.mjs

@@ -27,6 +27,7 @@ const CHANGE_SECTION_TITLES = ['Regressions', 'Improvements', 'Unchanged'];
 const NOT_COMPARED_SECTION_TITLE = 'Not Compared';
 const ALL_SECTION_TITLES = [...CHANGE_SECTION_TITLES, NOT_COMPARED_SECTION_TITLE];
 const SAMPLE_ENTITY_PREFIX = 'sample_entity_';
+const SAMPLE_ENTITY_WHOLE_POPULATION_SENTINEL = 'sample_entity_whole_population/sentinel';
 
 function readTextIfExists(filePath) {
   if (!filePath) return '';
@@ -108,6 +109,17 @@ function parseCriterionBodyToRows(body) {
   return rows;
 }
 
+function criterionBenchmarkNameForRow(row) {
+  if (row.group === 'sample_entity') {
+    return row.param ? `${row.bench}/${row.param}` : row.bench;
+  }
+  return row.param ? `${row.group}/${row.bench}/${row.param}` : `${row.group}/${row.bench}`;
+}
+
+function isInternalCriterionBenchmarkName(name) {
+  return name === SAMPLE_ENTITY_WHOLE_POPULATION_SENTINEL;
+}
+
 function parseNotComparedBodyToRows(body) {
   const lines = String(body || '').split(/\r?\n/);
   const rows = [];
@@ -250,7 +262,9 @@ function buildMarkdown({ hyperfineMd, criterionDir, groups }) {
     for (const t of CHANGE_SECTION_TITLES) {
       const body = extracted[t];
       if (body == null) continue;
-      const rows = parseCriterionBodyToRows(body);
+      const rows = parseCriterionBodyToRows(body).filter(
+        (row) => !isInternalCriterionBenchmarkName(criterionBenchmarkNameForRow(row)),
+      );
       bySection[t].push(...rows);
     }