diff --git a/ixa-bench/criterion/sample_entity_scaling.rs b/ixa-bench/criterion/sample_entity_scaling.rs index 403361c9..dd5663fe 100644 --- a/ixa-bench/criterion/sample_entity_scaling.rs +++ b/ixa-bench/criterion/sample_entity_scaling.rs @@ -5,6 +5,7 @@ use std::time::Instant; use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; use ixa::prelude::*; +use ixa::rand::Rng; define_rng!(SampleScalingRng); @@ -16,6 +17,9 @@ define_multi_property!((Species, Region), Mosquito); define_property!(struct Unindexed10(u8), Mosquito); const POPULATION_SIZES: [usize; 3] = [1_000, 10_000, 100_000]; +const WHOLE_POPULATION_SENTINEL: &str = "sentinel"; +const WHOLE_POPULATION_SENTINEL_UPPER: u32 = 100_000; +const WHOLE_POPULATION_SENTINEL_RESULTS_SIZE: usize = 0; // Shared place to stash "ns per sample" per (bench_name, size) type Results = Arc>>; @@ -69,6 +73,28 @@ fn print_scaling_summary(results: &BTreeMap<(String, usize), f64>, bench_name: & } } +fn print_whole_population_sentinel_summary( + results: &BTreeMap<(String, usize), f64>, + bench_name: &str, +) { + let sentinel_name = format!("{bench_name}/{WHOLE_POPULATION_SENTINEL}"); + let Some(sentinel_ns) = results.get(&(sentinel_name, WHOLE_POPULATION_SENTINEL_RESULTS_SIZE)) + else { + return; + }; + + eprintln!("\n=== Sentinel-relative summary: {bench_name} ==="); + for &size in &POPULATION_SIZES { + let Some(sample_ns) = results.get(&(bench_name.to_string(), size)) else { + continue; + }; + eprintln!( + " n={size:>7}: sample={sample_ns:.2} ns/sample, sentinel={sentinel_ns:.2} ns/sample, ratio={:.3}x", + sample_ns / sentinel_ns, + ); + } +} + // This is so we can do an O_n analysis of the benchmark fn bench_ns_per_sample(bencher: &mut criterion::Bencher, mut f: F) -> f64 where @@ -102,8 +128,9 @@ pub fn bench_sample_entity_whole_population(c: &mut Criterion, results: Results) group.bench_with_input(BenchmarkId::from_parameter(size), &size, |b, _| { let ns = bench_ns_per_sample(b, || { - let _: Option> = + let sampled: Option> = context.sample_entity(SampleScalingRng, Mosquito); + black_box(sampled); }); results @@ -113,6 +140,24 @@ pub fn bench_sample_entity_whole_population(c: &mut Criterion, results: Results) }); } + let sentinel_context = setup_context(WHOLE_POPULATION_SENTINEL_UPPER as usize); + group.bench_function(WHOLE_POPULATION_SENTINEL, |b| { + let ns = bench_ns_per_sample(b, || { + let sampled = sentinel_context.sample(SampleScalingRng, |rng| { + rng.random_range(0..WHOLE_POPULATION_SENTINEL_UPPER) + }); + black_box(sampled); + }); + + results.lock().unwrap().insert( + ( + format!("{bench_name}/{WHOLE_POPULATION_SENTINEL}"), + WHOLE_POPULATION_SENTINEL_RESULTS_SIZE, + ), + ns, + ); + }); + group.finish(); } @@ -205,6 +250,7 @@ fn sample_entity_scaling(c: &mut Criterion) { print_scaling_summary(&results, "sample_entity_single_property_indexed"); print_scaling_summary(&results, "sample_entity_multi_property_indexed"); print_scaling_summary(&results, "sample_entity_single_property_unindexed"); + print_whole_population_sentinel_summary(&results, "sample_entity_whole_population"); } criterion_group!(benches, sample_entity_scaling); diff --git a/scripts/bench_results.mjs b/scripts/bench_results.mjs index 859e09b6..8cb02c6a 100644 --- a/scripts/bench_results.mjs +++ b/scripts/bench_results.mjs @@ -25,6 +25,9 @@ import { fileURLToPath } from 'node:url'; const __filename = fileURLToPath(import.meta.url); const DEFAULT_MAX_INPUT_BYTES = 100 * 1024 * 1024; // 100 MiB +const SAMPLE_ENTITY_WHOLE_POPULATION_PREFIX = 'sample_entity_whole_population/'; +const SAMPLE_ENTITY_WHOLE_POPULATION_SENTINEL = 'sample_entity_whole_population/sentinel'; +const SAMPLE_ENTITY_WHOLE_POPULATION_SIZES = ['1000', '10000', '100000']; function maxInputBytes() { const raw = process.env.BENCH_RESULTS_MAX_INPUT_BYTES; @@ -150,6 +153,48 @@ function parseCriterionCompareLog(text) { return Array.from(resultsByName.values()); } +function hasFinitePositiveTriple(values) { + return ( + Array.isArray(values) && + values.length >= 3 && + values.slice(0, 3).every((value) => Number.isFinite(value) && value > 0) + ); +} + +function ratioText(value) { + return `${value.toFixed(3)}x`; +} + +function addSampleEntityWholePopulationSentinelRatios(criterionTimings) { + const byName = new Map(criterionTimings.map((entry) => [entry.name, entry])); + const sentinel = byName.get(SAMPLE_ENTITY_WHOLE_POPULATION_SENTINEL); + if (!hasFinitePositiveTriple(sentinel?.time_sec)) return criterionTimings; + + for (const size of SAMPLE_ENTITY_WHOLE_POPULATION_SIZES) { + const sampleName = `${SAMPLE_ENTITY_WHOLE_POPULATION_PREFIX}${size}`; + const sample = byName.get(sampleName); + if (!hasFinitePositiveTriple(sample?.time_sec)) continue; + + const ratio = [ + sample.time_sec[0] / sentinel.time_sec[2], + sample.time_sec[1] / sentinel.time_sec[1], + sample.time_sec[2] / sentinel.time_sec[0], + ]; + if (!ratio.every((value) => Number.isFinite(value) && value > 0)) continue; + + sample.relative_to_sentinel = { + kind: 'sentinel_ratio', + sentinel: SAMPLE_ENTITY_WHOLE_POPULATION_SENTINEL, + ratio, + ratio_text: ratio.map(ratioText), + method: + 'sample_time / sentinel_time; lower=sample_lower/sentinel_upper; estimate=sample_estimate/sentinel_estimate; upper=sample_upper/sentinel_lower', + }; + } + + return criterionTimings; +} + function parseHyperfineJson(hyperfineJson) { if (!hyperfineJson || !Array.isArray(hyperfineJson.results)) return []; @@ -224,7 +269,9 @@ function main() { const hyperfineTimings = parseHyperfineJson(hyperfineJson); const criterionCompareLog = readTextIfExists(criterionLogPath); - const criterionTimings = parseCriterionCompareLog(criterionCompareLog); + const criterionTimings = addSampleEntityWholePopulationSentinelRatios( + parseCriterionCompareLog(criterionCompareLog) + ); const payload = { schema: 1, diff --git a/scripts/format_bench_pr_comment.mjs b/scripts/format_bench_pr_comment.mjs index c03727a7..92dc3215 100644 --- a/scripts/format_bench_pr_comment.mjs +++ b/scripts/format_bench_pr_comment.mjs @@ -27,6 +27,7 @@ const CHANGE_SECTION_TITLES = ['Regressions', 'Improvements', 'Unchanged']; const NOT_COMPARED_SECTION_TITLE = 'Not Compared'; const ALL_SECTION_TITLES = [...CHANGE_SECTION_TITLES, NOT_COMPARED_SECTION_TITLE]; const SAMPLE_ENTITY_PREFIX = 'sample_entity_'; +const SAMPLE_ENTITY_WHOLE_POPULATION_SENTINEL = 'sample_entity_whole_population/sentinel'; function readTextIfExists(filePath) { if (!filePath) return ''; @@ -108,6 +109,17 @@ function parseCriterionBodyToRows(body) { return rows; } +function criterionBenchmarkNameForRow(row) { + if (row.group === 'sample_entity') { + return row.param ? `${row.bench}/${row.param}` : row.bench; + } + return row.param ? `${row.group}/${row.bench}/${row.param}` : `${row.group}/${row.bench}`; +} + +function isInternalCriterionBenchmarkName(name) { + return name === SAMPLE_ENTITY_WHOLE_POPULATION_SENTINEL; +} + function parseNotComparedBodyToRows(body) { const lines = String(body || '').split(/\r?\n/); const rows = []; @@ -250,7 +262,9 @@ function buildMarkdown({ hyperfineMd, criterionDir, groups }) { for (const t of CHANGE_SECTION_TITLES) { const body = extracted[t]; if (body == null) continue; - const rows = parseCriterionBodyToRows(body); + const rows = parseCriterionBodyToRows(body).filter( + (row) => !isInternalCriterionBenchmarkName(criterionBenchmarkNameForRow(row)), + ); bySection[t].push(...rows); } diff --git a/scripts/index.html b/scripts/index.html index a7e7c86b..68f1d284 100644 --- a/scripts/index.html +++ b/scripts/index.html @@ -215,6 +215,11 @@

IXA benchmark history

Show band + +