From 27f8e098ba32af5e2f403f59d639f275d353c9e9 Mon Sep 17 00:00:00 2001 From: Maggie Appleton <5599295+MaggieAppleton@users.noreply.github.com> Date: Sun, 18 Oct 2026 15:46:46 +0100 Subject: [PATCH 1/3] Add document records and rerun the coverage critique on decisions-v1 Fold in the cloud session's 41 git-file decision records (TC39 notes, ADRs/KEPs/TSC minutes, PEPs, Rust texts) as documents/decisions.jsonl, converted to the decisions-v1 record format and leak-checked against the refetched source documents. Add the cloud session's two coverage critiques and rerun their H1-H12 data pass on all 173 records, threads only and the small-team quota, with thread gaps coded into the v0 tag vocabulary and a blind reliability check (kappa 0.68). The holdout exclusion check is still pending. Co-Authored-By: Claude Opus 5.5 --- evals/datasets/decisions-v1/README.md | 89 +- .../decisions-v1/analysis/analyse_corpus.py | 409 ++ .../decisions-v1/analysis/corpus-metrics.json | 6215 +++++++++++++++++ .../analysis/coverage-critique.md | 237 + .../analysis/critique-coverage-data.md | 184 + .../analysis/critique-coverage-theory.md | 204 + .../analysis/tag-reliability.json | 365 + .../analysis/thread-gap-tags.json | 804 +++ .../decisions-v1/documents/convert_v0.py | 157 + .../decisions-v1/documents/decisions.jsonl | 41 + .../decisions-v1/documents/fetch_docs.py | 70 + .../decisions-v1/documents/validate.py | 103 + 12 files changed, 8867 insertions(+), 11 deletions(-) create mode 100644 evals/datasets/decisions-v1/analysis/analyse_corpus.py create mode 100644 evals/datasets/decisions-v1/analysis/corpus-metrics.json create mode 100644 evals/datasets/decisions-v1/analysis/coverage-critique.md create mode 100644 evals/datasets/decisions-v1/analysis/critique-coverage-data.md create mode 100644 evals/datasets/decisions-v1/analysis/critique-coverage-theory.md create mode 100644 evals/datasets/decisions-v1/analysis/tag-reliability.json create mode 100644 evals/datasets/decisions-v1/analysis/thread-gap-tags.json create mode 100644 evals/datasets/decisions-v1/documents/convert_v0.py create mode 100644 evals/datasets/decisions-v1/documents/decisions.jsonl create mode 100644 evals/datasets/decisions-v1/documents/fetch_docs.py create mode 100644 evals/datasets/decisions-v1/documents/validate.py diff --git a/evals/datasets/decisions-v1/README.md b/evals/datasets/decisions-v1/README.md index 506fb6778..660770d21 100644 --- a/evals/datasets/decisions-v1/README.md +++ b/evals/datasets/decisions-v1/README.md @@ -1,7 +1,8 @@ # decisions-v1 -A corpus of 127 real software decisions debated on public GitHub threads. It -has two jobs: +A corpus of 127 real software decisions debated on public GitHub threads, plus +41 decision records taken from complete documents (ADRs, KEPs, meeting minutes, +PEPs and RFC texts). It has two jobs: 1. **Coverage critique** for Chopin's questions feature. Each decision record says which question shape it needed (explore, choice, tune, check), which @@ -10,15 +11,17 @@ has two jobs: [`../v2/discussions`](../v2/discussions/README.md), with 2–4 cutoffs at meaningful moments. -| File | Contents | -| ------------------------------------ | ----------------------------------------------------------------------------- | -| [`cases/*.json`](cases) | 127 adapted threads, 8,367 paraphrased events | -| [`decisions.jsonl`](decisions.jsonl) | 132 decision records; a thread may hold more than one | -| [`manifest.json`](manifest.json) | Path, SHA-256, bytes, source URL, retrieval time and cutoff purposes per case | -| [`stats.json`](stats.json) | Shape, flow and gap distributions, plus process shares | -| [`candidates.json`](candidates.json) | Every one of the 192 screened candidates and its outcome | -| [`validate.py`](validate.py) | Schema, enum, identity, manifest and licensing-leak checks | -| [`tools/`](tools) | The fetcher, normaliser, assembler, packager and annotation guide | +| File | Contents | +| ------------------------------------ | -------------------------------------------------------------------------------------------- | +| [`cases/*.json`](cases) | 127 adapted threads, 8,367 paraphrased events | +| [`decisions.jsonl`](decisions.jsonl) | 132 decision records; a thread may hold more than one | +| [`manifest.json`](manifest.json) | Path, SHA-256, bytes, source URL, retrieval time and cutoff purposes per case | +| [`stats.json`](stats.json) | Shape, flow and gap distributions, plus process shares | +| [`candidates.json`](candidates.json) | Every one of the 192 screened candidates and its outcome | +| [`validate.py`](validate.py) | Schema, enum, identity, manifest and licensing-leak checks | +| [`tools/`](tools) | The fetcher, normaliser, assembler, packager and annotation guide | +| [`documents/`](documents) | 41 decision records from complete documents, with no case files ([below](#document-records)) | +| [`analysis/`](analysis) | The coverage critique, rerun on all 173 records ([below](#analysis)) | ## Provenance @@ -76,6 +79,67 @@ thread URL in that snapshot's conversation corpus was also checked, and none matched. The holdout URLs are deliberately not recorded here. Rerun `tools/holdout_check.py ` after adding cases. +## Document records + +[`documents/decisions.jsonl`](documents/decisions.jsonl) holds 41 decision +records an earlier cloud session wrote from complete files in public +repositories: + +| Source | Records | +| --------------------------------------------------------- | ------: | +| TC39 meeting notes | 15 | +| Engineering ADRs, Kubernetes KEPs and Node.js TSC minutes | 16 | +| Python PEPs | 6 | +| Rust RFC text and lang-team notes | 4 | + +They are the cloud corpus v0 records whose `fidelity.method` was `git-file`, +excluding its `chopin-v2` collector. Its webfetch and mixed records were +dropped, because this corpus refetched those threads in full. None shares a +source URL with the thread records. + +These records come from documents, not threads, so they have no case files and +no cutoffs. [`documents/convert_v0.py`](documents/convert_v0.py) converts them +to this corpus's record format: + +- `caseId` is `null`, `collector` is `cloud-git` and `fidelity.method` is `git-file`. +- `fitsFlows` loses v0's `F1`-style prefixes. +- Free-text deciders map onto the `decider` enum. The original wording is kept in + `notes`. +- `process.reactionsTotal` is `null`. `process.decidedOffPlatform` defaults to + `false`, because v0 had no such field. +- Each gap becomes `cluster: detail`. The cluster comes from the gap's v0 tag, + using this corpus's cluster names where they match. v0's extra fields + (`gapTags`, `personaRelevance`, `contentKind`, `kruchtenKind`, `codingPass`) + are dropped. +- Counts a document cannot supply (`participants.count`, `durationDays`, + `comments`) stay `null` where v0 left them `null`. + +The records are paraphrases written by the cloud session. They were checked the +same way as the thread records: + +- [`documents/fetch_docs.py`](documents/fetch_docs.py) fetches all 72 distinct + source documents through `gh api` into the git-ignored + `.cache/decisions/docs/`. PEPs are read from `python/peps`. +- [`documents/validate.py`](documents/validate.py) runs the main validator's + record checks, allowing only the differences above. It also fails on any run + of 8 or more words copied from the source documents. + +Two v0 phrases failed the leak check and are reworded in `convert_v0.py`. + +## Analysis + +[`analysis/`](analysis) holds the two critiques the cloud session wrote against +v0, kept unchanged. It also holds a rerun on this corpus: + +- [`coverage-critique.md`](analysis/coverage-critique.md): H1–H12 retested, and + the gaps re-ranked, for the whole corpus, threads only and the small-team quota. +- [`analyse_corpus.py`](analysis/analyse_corpus.py): produces + [`corpus-metrics.json`](analysis/corpus-metrics.json). +- [`thread-gap-tags.json`](analysis/thread-gap-tags.json): codes each thread gap + into v0's tag vocabulary, so the two sources count alike. +- [`tag-reliability.json`](analysis/tag-reliability.json): a blind recode of + 60 document gaps against v0's tags (κ = 0.68). + ## Licensing approach GitHub's terms do not clearly permit republishing third-party comments, so @@ -192,6 +256,9 @@ python3 -I evals/datasets/decisions-v1/tools/normalise.py python3 -I evals/datasets/decisions-v1/tools/assemble.py # needs .cache/decisions/ann/ python3 -I evals/datasets/decisions-v1/tools/package.py python3 -I evals/datasets/decisions-v1/validate.py +python3 -I evals/datasets/decisions-v1/documents/fetch_docs.py # cached +python3 -I evals/datasets/decisions-v1/documents/validate.py +python3 -I evals/datasets/decisions-v1/analysis/analyse_corpus.py bun test evals/datasets # structural checks, no cache needed ``` diff --git a/evals/datasets/decisions-v1/analysis/analyse_corpus.py b/evals/datasets/decisions-v1/analysis/analyse_corpus.py new file mode 100644 index 000000000..910358343 --- /dev/null +++ b/evals/datasets/decisions-v1/analysis/analyse_corpus.py @@ -0,0 +1,409 @@ +"""Data pass for the coverage critique on the combined decisions-v1 corpus. + +Usage: python3 -I evals/datasets/decisions-v1/analysis/analyse_corpus.py + +Adapted from the cloud session's v0 script (critique-coverage-data.md). Tests +H1–H12 from critique-coverage-theory.md §5 and re-ranks gaps over: + all 132 thread records + 41 document records + threads the 132 thread records (decisions.jsonl) + documents the 41 document records (documents/decisions.jsonl) + small thread records whose case is in the small-team quota +Writes analysis/corpus-metrics.json. Brackets are Wilson 95% intervals. + +Every gap carries one tag from v0's controlled vocabulary. Document gaps keep +the v0 coder's tag, recovered from the cluster name convert_v0.py gave it. Thread +gaps were coded into the same vocabulary afterwards (thread-gap-tags.json; +see coverage-critique.md for the method and the agreement check, tag-reliability.json). +""" + +import importlib.util +import json +import math +import pathlib +import re +import statistics +from collections import Counter + +HERE = pathlib.Path(__file__).resolve().parent +DATASET = HERE.parent +_spec = importlib.util.spec_from_file_location("package", DATASET / "tools" / "package.py") +package = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(package) + + +def load(path): + return [json.loads(l) for l in path.read_text().splitlines() if l.strip()] + + +threads = load(DATASET / "decisions.jsonl") +documents = load(DATASET / "documents" / "decisions.jsonl") +quota = {c["id"]: c["sourceCluster"] for c in json.loads((DATASET / "manifest.json").read_text())["cases"]} + +# ---------------------------------------------------------------- gap tags + +_conv = importlib.util.spec_from_file_location("convert_v0", DATASET / "documents" / "convert_v0.py") +convert_v0 = importlib.util.module_from_spec(_conv) +_conv.loader.exec_module(convert_v0) +# Cluster name -> v0 tag. The convert_v0 mapping is one-to-one apart from v0's `other` split. +DOC_TAG = {v: k for k, v in convert_v0.CLUSTER.items()} +DOC_TAG.update({"hidden history": "other:hidden-history", "deadline": "other:deadline", "other": "other:misc"}) +THREAD_TAGS = json.loads((HERE / "thread-gap-tags.json").read_text()) +OTHER_RULES = convert_v0.OTHER + + +def thread_tag(tag, gap): + if tag != "other": + return tag + for name, rx in OTHER_RULES: + if rx.search(gap): + return "other:" + name.replace(" ", "-") + return "other:misc" + + +def tags_of(r): + gaps = r["chopinFit"]["gaps"] + if r["_source"] == "document": + return [DOC_TAG[g.split(":", 1)[0]] for g in gaps] + tags = THREAD_TAGS[r["id"]] + if len(tags) != len(gaps): + raise ValueError(f"{r['id']}: {len(tags)} tags for {len(gaps)} gaps") + return [thread_tag(t, g) for t, g in zip(tags, gaps)] + + +FORMAL_REPOS = re.compile(r"^(tc39|rust-lang|python|golang)/", re.I) +for r in threads: + r["_source"] = "thread" + r["_quota"] = quota.get(r["caseId"]) + r["_formal"] = r["_quota"] == "formal-process" + r["_standards"] = r["repo"].lower().startswith("tc39/") +for r in documents: + r["_source"] = "document" + r["_quota"] = None + r["_formal"] = bool(FORMAL_REPOS.match(r["repo"])) or "peps.python.org" in r["sources"][0]["url"] + r["_standards"] = r["repo"].lower().startswith("tc39/") +recs = threads + documents +for r in recs: + r["_taglist"] = tags_of(r) + r["_tags"] = set(r["_taglist"]) + r["_kinds"] = {s["kind"] for s in r["sources"]} + +STRATA = { + "all": recs, + "threads": threads, + "documents": documents, + "small": [r for r in threads if r["_quota"] == "small-team"], +} + +# ---------------------------------------------------------------- helpers (from v0) + + +def wilson(k, n, z=1.96): + if n == 0: + return None + p = k / n + d = 1 + z * z / n + c = (p + z * z / (2 * n)) / d + h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d + return [round(max(0, c - h), 3), round(min(1, c + h), 3)] + + +def prop(k, n): + return {"k": k, "n": n, "p": round(k / n, 3) if n else None, "ci95": wilson(k, n)} + + +def frac(rs, pred): + return prop(sum(1 for r in rs if pred(r)), len(rs)) + + +def shape(r): + return r["chopinFit"]["shape"] + + +def describe(rs): + pc = [r["participants"]["count"] for r in rs if r["participants"]["count"] is not None] + dur = [r["process"]["durationDays"] for r in rs if r["process"]["durationDays"] is not None] + return { + "n": len(rs), + "quota": dict(Counter(r["_quota"] or "document" for r in rs).most_common()), + "domain": dict(Counter(r["domain"] for r in rs).most_common()), + "participants_known": len(pc), + "participants_median": statistics.median(pc) if pc else None, + "participants_le3": sum(1 for c in pc if c <= 3), + "duration_median_days": statistics.median(dur) if dur else None, + "decided": frac(rs, lambda r: r["outcome"]["decided"]), + "how": dict(Counter(r["outcome"]["how"] for r in rs).most_common()), + "decider": dict(Counter(r["participants"]["decider"] for r in rs).most_common()), + "formal_share": frac(rs, lambda r: r["_formal"]), + "truncated": frac(rs, lambda r: r["fidelity"]["truncated"]), + } + +# ---------------------------------------------------------------- hypotheses (v0 definitions) + + +def h1(rs): + return { + "mixed_or_none": frac(rs, lambda r: shape(r) in ("mixed", "none")), + "has_secondaryShape": frac(rs, lambda r: r["chopinFit"].get("secondaryShape")), + "shape_dist": dict(Counter(shape(r) for r in rs).most_common()), + "primary_secondary_pairs": dict(Counter(f'{shape(r)}+{r["chopinFit"].get("secondaryShape")}' for r in rs).most_common(8)), + } + + +def h2(rs): + uix = [r for r in rs if r["domain"] in ("ui-ux", "product-behaviour")] + c = Counter(shape(r) for r in rs) + return { + "choice": frac(rs, lambda r: shape(r) == "choice"), + "tune": frac(rs, lambda r: shape(r) == "tune"), + "choice_is_modal": bool(c) and c.most_common(1)[0][0] == "choice", + "uiux_product_n": len(uix), + "uiux_product_tune": frac(uix, lambda r: shape(r) == "tune"), + "uiux_product_tune_primary_or_secondary": frac(uix, lambda r: "tune" in (shape(r), r["chopinFit"].get("secondaryShape"))), + "tune_primary_or_secondary": frac(rs, lambda r: "tune" in (shape(r), r["chopinFit"].get("secondaryShape"))), + "check_primary_or_secondary": frac(rs, lambda r: "check" in (shape(r), r["chopinFit"].get("secondaryShape"))), + "explore_primary_or_secondary": frac(rs, lambda r: "explore" in (shape(r), r["chopinFit"].get("secondaryShape"))), + } + + +def h3(rs): + ch = [r for r in rs if shape(r) == "choice"] + lens = [len(r["options"]) for r in ch] + return { + "n_choice": len(ch), + "options_median": statistics.median(lens) if lens else None, + "options_ge4": frac(ch, lambda r: len(r["options"]) >= 4), + "criteria_nonempty": frac(ch, lambda r: bool(r["criteria"])), + "composite_answer": frac(ch, lambda r: sum(o.get("fate") == "chosen" for o in r["options"]) >= 2 or any(o.get("fate") in ("partly", "merged") for o in r["options"])), + "has_deferred_option": frac(ch, lambda r: any(o.get("fate") == "deferred" for o in r["options"])), + } + + +def h4(rs): + d = [r for r in rs if r["outcome"]["decided"] and len(r["options"]) >= 2] + return { + "order_guaranteed": False, + "first_option_chosen": frac(d, lambda r: r["options"][0].get("fate") == "chosen"), + "first_option_chosen_by_source": {s: frac([r for r in d if r["_source"] == s], lambda r: r["options"][0].get("fate") == "chosen") for s in ("thread", "document")}, + } + + +def h5(rs): + und = [r for r in rs if not r["outcome"]["decided"]] + return { + "deferred_flag": frac(rs, lambda r: r["process"]["deferred"]), + "undecided_and_deferred": frac(rs, lambda r: (not r["outcome"]["decided"]) and r["process"]["deferred"]), + "parked_any": frac(rs, lambda r: ((not r["outcome"]["decided"]) and r["process"]["deferred"]) or bool(r["_tags"] & {"deferred", "stale-idle"})), + "undecided_n": len(und), + "undecided_deferred": sum(1 for r in und if r["process"]["deferred"]), + "undecided_abandoned": sum(1 for r in und if r["outcome"]["how"] == "abandoned"), + "decided_but_deferred_flag": sum(1 for r in rs if r["outcome"]["decided"] and r["process"]["deferred"]), + } + + +STRUCT = {"coupled", "blocking", "staged"} + + +def h6(rs): + rp = [r for r in rs if r["_kinds"] & {"rfc", "proposal"}] + return { + "subq_nonempty": frac(rs, lambda r: bool(r["process"]["subQuestions"])), + "struct_tag": frac(rs, lambda r: bool(r["_tags"] & (STRUCT | {"nested"}))), + "rfc_or_proposal_n": len(rp), + "rfc_or_proposal_struct_tag": frac(rp, lambda r: bool(r["_tags"] & (STRUCT | {"nested"}))), + "follow_up_structure": frac(rs, lambda r: bool(r["_tags"] & {"nested", "staged", "consequences"})), + } + + +def h7(rs): + nc = [r for r in rs if shape(r) != "check"] + return { + "n_noncheck": len(nc), + "research_or_spike": frac(nc, lambda r: r["process"]["research"] != "none" or r["process"]["spike"] != "none"), + "strict_(research_beyond_links_or_spike)": frac(nc, lambda r: r["process"]["research"] not in ("none", "links") or r["process"]["spike"] != "none"), + "spike_any": frac(nc, lambda r: r["process"]["spike"] != "none"), + "strict_excluding_implementation_pr": frac(nc, lambda r: r["process"]["research"] not in ("none", "links") or r["process"]["spike"] not in ("none", "implementation-pr")), + "spike_dist": dict(Counter(r["process"]["spike"] for r in nc).most_common()), + "evidence_tag": frac(nc, lambda r: "evidence-not-check" in r["_tags"]), + "check_secondary": frac(nc, lambda r: r["chopinFit"].get("secondaryShape") == "check"), + } + + +def h8(rs): + d = [r for r in rs if r["outcome"]["decided"]] + return { + "n_decided": len(d), + "reversibility_stated": frac(d, lambda r: r["outcome"]["reversibility"] in ("stated-easy", "stated-hard")), + "confidence_stated": frac(d, lambda r: r["outcome"]["confidence"] == "stated"), + "confidence_stated_by_source": {s: frac([r for r in d if r["_source"] == s], lambda r: r["outcome"]["confidence"] == "stated") for s in ("thread", "document")}, + "provisional_or_conditional_tag": frac(rs, lambda r: bool(r["_tags"] & {"provisional", "conditional"})), + } + + +def h9(rs): + d3 = [r for r in rs if r["outcome"]["decided"] and (r["participants"]["count"] or 0) >= 3] + d = [r for r in rs if r["outcome"]["decided"]] + return { + "decided_ge3_n": len(d3), + "dissent_decided_ge3": frac(d3, lambda r: r["outcome"]["dissent"]), + "dissent_all_decided": frac(d, lambda r: r["outcome"]["dissent"]), + } + + +def h10(rs): + rv = [r for r in rs if r["process"]["reopened"] or r["process"]["superseded"]] + return { + "reopened_or_superseded": frac(rs, lambda r: r["process"]["reopened"] or r["process"]["superseded"]), + "reopened": frac(rs, lambda r: r["process"]["reopened"]), + "superseded": frac(rs, lambda r: r["process"]["superseded"]), + "of_those_supersede_or_revert_tag": frac(rv, lambda r: bool(r["_tags"] & {"supersede", "revert"})), + "supersede_or_revert_tag_any": frac(rs, lambda r: bool(r["_tags"] & {"supersede", "revert"})), + } + + +def h11(rs): + return { + "pr_source_and_impl_pr": frac(rs, lambda r: "pr" in r["_kinds"] and r["process"]["spike"] == "implementation-pr"), + "impl_pr_any_source": frac(rs, lambda r: r["process"]["spike"] == "implementation-pr"), + "pr_source": frac(rs, lambda r: "pr" in r["_kinds"]), + "implicit_tag": frac(rs, lambda r: "implicit" in r["_tags"]), + "implementedIn_nonempty": frac(rs, lambda r: bool(r["outcome"]["implementedIn"])), + } + + +SINGLE = {"named maintainer", "BDFL"} + + +def h12(rs): + le3 = [r for r in rs if r["participants"]["count"] is not None and r["participants"]["count"] <= 3] + nonstd = [r for r in rs if not r["_standards"]] + nonformal = [r for r in rs if not r["_formal"]] + vp = lambda r: r["process"]["poll"] or r["outcome"]["how"] == "vote" + return { + "le3_n": len(le3), + "le3_ids": [r["id"] for r in le3], + "le3_single_decider": frac(le3, lambda r: r["participants"]["decider"] in SINGLE), + "single_decider": frac(rs, lambda r: r["participants"]["decider"] in SINGLE), + "maintainer_decision_how": frac(rs, lambda r: r["outcome"]["how"] == "maintainer-decision"), + "team_or_consensus_decider": frac(rs, lambda r: r["participants"]["decider"] in ("team", "consensus")), + "decider_unclear": frac(rs, lambda r: r["participants"]["decider"] == "unclear"), + "vote_or_poll_nonstandards": frac(nonstd, vp), + "vote_or_poll_nonformal": frac(nonformal, vp), + "vote_or_poll_all": frac(rs, vp), + } + + +HYP = {"H1": h1, "H2": h2, "H3": h3, "H4": h4, "H5": h5, "H6": h6, + "H7": h7, "H8": h8, "H9": h9, "H10": h10, "H11": h11, "H12": h12} + +# ---------------------------------------------------------------- gaps + + +def tag_table(): + names = Counter(t for r in recs for t in r["_tags"]) + out = {} + for name, _ in names.most_common(): + row = {s: frac(rs, lambda r, n=name: n in r["_tags"]) for s, rs in STRATA.items()} + row["entries_total"] = sum(r["_taglist"].count(name) for r in recs) + hits = [r for r in recs if name in r["_tags"]] + row["formal_share_of_hits"] = frac(hits, lambda r: r["_formal"]) + row["examples"] = [{"id": r["id"], "source": r["_source"], "url": r["sources"][0]["url"], + "gap": next(g for t, g in zip(r["_taglist"], r["chopinFit"]["gaps"]) if t == name)[:200]} + for r in sorted(hits, key=lambda r: (r["_quota"] != "small-team", r["_source"] != "thread", r["id"]))[:4]] + out[name] = row + return out + + +def reliability(): + """Blind recoding of 60 document gaps against the v0 coder's tags (Cohen's kappa).""" + data = json.loads((HERE / "tag-reliability.json").read_text()) + a = [x["v0"] for x in data["items"]] + b = [x["recoded"] for x in data["items"]] + n = len(a) + po = sum(x == y for x, y in zip(a, b)) / n + ca, cb = Counter(a), Counter(b) + pe = sum(ca[k] * cb[k] for k in set(a) | set(b)) / (n * n) + return {"n": n, "agree": sum(x == y for x, y in zip(a, b)), "po": round(po, 3), "pe": round(pe, 3), + "kappa": round((po - pe) / (1 - pe), 3), + "confusions": dict(Counter(f"{x}->{y}" for x, y in zip(a, b) if x != y).most_common(10))} + + +# The theory pass's gap groups (critique-coverage-theory.md §3), as concept sets. +GROUPS = { + "G7 follow-ups: nested|staged|consequences": lambda r, t: bool(t & {"nested", "staged", "consequences"}), + "G9' composite answer (>=2 chosen or partly/merged)": lambda r, t: sum(o.get("fate") == "chosen" for o in r["options"]) >= 2 or any(o.get("fate") in ("partly", "merged") for o in r["options"]), + "G2 parked: deferred|stale-idle or undecided+deferred": lambda r, t: bool(t & {"deferred", "stale-idle"}) or ((not r["outcome"]["decided"]) and r["process"]["deferred"]), + "G6 evidence beyond check": lambda r, t: "evidence-not-check" in t, + "G3 decided elsewhere: venue-split|implicit|ratify": lambda r, t: bool(t & {"venue-split", "implicit", "ratify"}), + "P8 external authority": lambda r, t: "external-authority" in t, + "G5 provisional|conditional": lambda r, t: bool(t & {"provisional", "conditional"}), + "G10 supersede|revert": lambda r, t: bool(t & {"supersede", "revert"}), + "G4 coupled|blocking": lambda r, t: bool(t & {"coupled", "blocking"}), + "P10 crowd signal (votes as evidence)": lambda r, t: "crowd-signal" in t, + "G12 delegate|approve": lambda r, t: bool(t & {"delegate", "approve"}), + "P6 consensus mechanics": lambda r, t: "consensus-mechanics" in t, + "N reframe": lambda r, t: "reframe" in t, + "G8 convention": lambda r, t: "convention" in t, + "N visual comparison": lambda r, t: "visual-comparison" in t, + "N rejected-content": lambda r, t: "rejected-content" in t, + "N consequences": lambda r, t: "consequences" in t, + "G9 rank|partition": lambda r, t: bool(t & {"rank", "partition"}), + "G11 moot": lambda r, t: "moot" in t, + "C deadline": lambda r, t: "other:deadline" in t, + "X hidden history (corpus artefact)": lambda r, t: "other:hidden-history" in t, + "P7 formal vote": lambda r, t: "vote" in t, +} + + +def group_table(): + out = {} + for g, pred in GROUPS.items(): + row = {s: frac(rs, lambda r: pred(r, r["_tags"])) for s, rs in STRATA.items()} + hits = [r for r in recs if pred(r, r["_tags"])] + row["formal_share_of_hits"] = frac(hits, lambda r: r["_formal"]) + row["examples"] = [r["id"] for r in sorted(hits, key=lambda r: (r["_quota"] != "small-team", r["_source"] != "thread", r["id"]))[:4]] + out[g] = row + return out + + +def flows(): + return {s: {k: prop(v, len(rs)) for k, v in sorted(Counter(f for r in rs for f in r["chopinFit"]["fitsFlows"]).items())} + for s, rs in STRATA.items()} + + +metrics = { + "meta": { + "n": len(recs), + "strata": {s: len(rs) for s, rs in STRATA.items()}, + "ci": "Wilson 95% interval", + }, + "describe": {s: describe(rs) for s, rs in STRATA.items()}, + "hypotheses": {h: {s: f(rs) for s, rs in STRATA.items()} for h, f in HYP.items()}, + "tags": tag_table(), + "tag_reliability": reliability(), + "gap_groups": group_table(), + "flows": flows(), +} +(HERE / "corpus-metrics.json").write_text(json.dumps(metrics, indent="\t", ensure_ascii=False) + "\n") + + +def p(x): + return f'{x["k"]}/{x["n"]}={x["p"]}' if x and x["n"] else "n/a" + + +if __name__ == "__main__": + print({s: len(rs) for s, rs in STRATA.items()}) + for s in STRATA: + d = metrics["describe"][s] + print(s, {k: (p(v) if isinstance(v, dict) and "k" in v else v) for k, v in d.items() if k not in ("domain",)}) + for h, by in metrics["hypotheses"].items(): + print("==", h) + for s in STRATA: + print(" ", s, {k: (p(v) if isinstance(v, dict) and "k" in v else v) for k, v in by[s].items() + if not (isinstance(v, dict) and "k" not in v) or k in ("shape_dist", "primary_secondary_pairs")}) + print("== groups") + for g, row in metrics["gap_groups"].items(): + print(f' {g:52s}', " ".join(f'{s} {p(row[s]):12s}' for s in STRATA), "| formal", p(row["formal_share_of_hits"])) + print("== reliability", {k: v for k, v in metrics["tag_reliability"].items()}) + print("== tags") + for c, row in metrics["tags"].items(): + print(f' {c:42s}', " ".join(f'{s} {p(row[s]):12s}' for s in STRATA)) diff --git a/evals/datasets/decisions-v1/analysis/corpus-metrics.json b/evals/datasets/decisions-v1/analysis/corpus-metrics.json new file mode 100644 index 000000000..648982036 --- /dev/null +++ b/evals/datasets/decisions-v1/analysis/corpus-metrics.json @@ -0,0 +1,6215 @@ +{ + "meta": { + "n": 173, + "strata": { + "all": 173, + "threads": 132, + "documents": 41, + "small": 11 + }, + "ci": "Wilson 95% interval" + }, + "describe": { + "all": { + "n": 173, + "quota": { + "document": 41, + "product-ui": 36, + "pr-debate": 28, + "formal-process": 26, + "eng-issue": 20, + "small-team": 11, + "open-choice": 11 + }, + "domain": { + "ui-ux": 32, + "api-design": 25, + "product-behaviour": 22, + "architecture": 15, + "process-governance": 15, + "language-design": 14, + "compatibility": 12, + "tooling": 9, + "security": 8, + "naming": 8, + "data-model": 8, + "infra": 3, + "dependency": 2 + }, + "participants_known": 155, + "participants_median": 15, + "participants_le3": 3, + "duration_median_days": 114.0, + "decided": { + "k": 148, + "n": 173, + "p": 0.855, + "ci95": [ + 0.795, + 0.9 + ] + }, + "how": { + "maintainer-decision": 83, + "consensus": 31, + "unclear": 23, + "default/no-action": 13, + "fcp": 12, + "abandoned": 7, + "vote": 4 + }, + "decider": { + "team": 63, + "named maintainer": 49, + "consensus": 26, + "unclear": 21, + "BDFL": 8, + "vote": 6 + }, + "formal_share": { + "k": 51, + "n": 173, + "p": 0.295, + "ci95": [ + 0.232, + 0.367 + ] + }, + "truncated": { + "k": 5, + "n": 173, + "p": 0.029, + "ci95": [ + 0.012, + 0.066 + ] + } + }, + "threads": { + "n": 132, + "quota": { + "product-ui": 36, + "pr-debate": 28, + "formal-process": 26, + "eng-issue": 20, + "small-team": 11, + "open-choice": 11 + }, + "domain": { + "ui-ux": 32, + "product-behaviour": 22, + "api-design": 22, + "architecture": 11, + "compatibility": 8, + "language-design": 8, + "data-model": 8, + "process-governance": 6, + "tooling": 5, + "naming": 5, + "security": 3, + "infra": 1, + "dependency": 1 + }, + "participants_known": 132, + "participants_median": 18.0, + "participants_le3": 3, + "duration_median_days": 188.0, + "decided": { + "k": 117, + "n": 132, + "p": 0.886, + "ci95": [ + 0.821, + 0.93 + ] + }, + "how": { + "maintainer-decision": 78, + "consensus": 18, + "fcp": 12, + "default/no-action": 10, + "abandoned": 7, + "unclear": 4, + "vote": 3 + }, + "decider": { + "named maintainer": 48, + "team": 48, + "unclear": 15, + "consensus": 12, + "BDFL": 6, + "vote": 3 + }, + "formal_share": { + "k": 26, + "n": 132, + "p": 0.197, + "ci95": [ + 0.138, + 0.273 + ] + }, + "truncated": { + "k": 2, + "n": 132, + "p": 0.015, + "ci95": [ + 0.004, + 0.054 + ] + } + }, + "documents": { + "n": 41, + "quota": { + "document": 41 + }, + "domain": { + "process-governance": 9, + "language-design": 6, + "security": 5, + "architecture": 4, + "tooling": 4, + "compatibility": 4, + "naming": 3, + "api-design": 3, + "infra": 2, + "dependency": 1 + }, + "participants_known": 23, + "participants_median": 10, + "participants_le3": 0, + "duration_median_days": 21.5, + "decided": { + "k": 31, + "n": 41, + "p": 0.756, + "ci95": [ + 0.607, + 0.862 + ] + }, + "how": { + "unclear": 19, + "consensus": 13, + "maintainer-decision": 5, + "default/no-action": 3, + "vote": 1 + }, + "decider": { + "team": 15, + "consensus": 14, + "unclear": 6, + "vote": 3, + "BDFL": 2, + "named maintainer": 1 + }, + "formal_share": { + "k": 25, + "n": 41, + "p": 0.61, + "ci95": [ + 0.457, + 0.743 + ] + }, + "truncated": { + "k": 3, + "n": 41, + "p": 0.073, + "ci95": [ + 0.025, + 0.194 + ] + } + }, + "small": { + "n": 11, + "quota": { + "small-team": 11 + }, + "domain": { + "product-behaviour": 6, + "language-design": 2, + "compatibility": 1, + "tooling": 1, + "architecture": 1 + }, + "participants_known": 11, + "participants_median": 17, + "participants_le3": 0, + "duration_median_days": 1187, + "decided": { + "k": 10, + "n": 11, + "p": 0.909, + "ci95": [ + 0.623, + 0.984 + ] + }, + "how": { + "maintainer-decision": 10, + "default/no-action": 1 + }, + "decider": { + "named maintainer": 7, + "unclear": 2, + "BDFL": 2 + }, + "formal_share": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "truncated": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + } + } + }, + "hypotheses": { + "H1": { + "all": { + "mixed_or_none": { + "k": 16, + "n": 173, + "p": 0.092, + "ci95": [ + 0.058, + 0.145 + ] + }, + "has_secondaryShape": { + "k": 141, + "n": 173, + "p": 0.815, + "ci95": [ + 0.751, + 0.866 + ] + }, + "shape_dist": { + "choice": 124, + "mixed": 16, + "explore": 14, + "check": 12, + "tune": 7 + }, + "primary_secondary_pairs": { + "choice+check": 36, + "choice+explore": 35, + "choice+None": 32, + "choice+tune": 21, + "explore+choice": 12, + "check+choice": 11, + "mixed+choice": 9, + "tune+choice": 5 + } + }, + "threads": { + "mixed_or_none": { + "k": 11, + "n": 132, + "p": 0.083, + "ci95": [ + 0.047, + 0.143 + ] + }, + "has_secondaryShape": { + "k": 117, + "n": 132, + "p": 0.886, + "ci95": [ + 0.821, + 0.93 + ] + }, + "shape_dist": { + "choice": 93, + "mixed": 11, + "check": 11, + "explore": 10, + "tune": 7 + }, + "primary_secondary_pairs": { + "choice+explore": 32, + "choice+check": 27, + "choice+tune": 19, + "choice+None": 15, + "check+choice": 10, + "explore+choice": 8, + "mixed+choice": 7, + "tune+choice": 5 + } + }, + "documents": { + "mixed_or_none": { + "k": 5, + "n": 41, + "p": 0.122, + "ci95": [ + 0.053, + 0.255 + ] + }, + "has_secondaryShape": { + "k": 24, + "n": 41, + "p": 0.585, + "ci95": [ + 0.434, + 0.722 + ] + }, + "shape_dist": { + "choice": 31, + "mixed": 5, + "explore": 4, + "check": 1 + }, + "primary_secondary_pairs": { + "choice+None": 17, + "choice+check": 9, + "explore+choice": 4, + "choice+explore": 3, + "choice+tune": 2, + "mixed+choice": 2, + "mixed+check": 2, + "check+choice": 1 + } + }, + "small": { + "mixed_or_none": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "has_secondaryShape": { + "k": 10, + "n": 11, + "p": 0.909, + "ci95": [ + 0.623, + 0.984 + ] + }, + "shape_dist": { + "choice": 8, + "explore": 1, + "check": 1, + "tune": 1 + }, + "primary_secondary_pairs": { + "choice+check": 3, + "choice+tune": 2, + "choice+explore": 2, + "explore+choice": 1, + "check+choice": 1, + "choice+None": 1, + "tune+explore": 1 + } + } + }, + "H2": { + "all": { + "choice": { + "k": 124, + "n": 173, + "p": 0.717, + "ci95": [ + 0.645, + 0.779 + ] + }, + "tune": { + "k": 7, + "n": 173, + "p": 0.04, + "ci95": [ + 0.02, + 0.081 + ] + }, + "choice_is_modal": true, + "uiux_product_n": 54, + "uiux_product_tune": { + "k": 7, + "n": 54, + "p": 0.13, + "ci95": [ + 0.064, + 0.244 + ] + }, + "uiux_product_tune_primary_or_secondary": { + "k": 24, + "n": 54, + "p": 0.444, + "ci95": [ + 0.32, + 0.576 + ] + }, + "tune_primary_or_secondary": { + "k": 31, + "n": 173, + "p": 0.179, + "ci95": [ + 0.129, + 0.243 + ] + }, + "check_primary_or_secondary": { + "k": 55, + "n": 173, + "p": 0.318, + "ci95": [ + 0.253, + 0.391 + ] + }, + "explore_primary_or_secondary": { + "k": 51, + "n": 173, + "p": 0.295, + "ci95": [ + 0.232, + 0.367 + ] + } + }, + "threads": { + "choice": { + "k": 93, + "n": 132, + "p": 0.705, + "ci95": [ + 0.622, + 0.776 + ] + }, + "tune": { + "k": 7, + "n": 132, + "p": 0.053, + "ci95": [ + 0.026, + 0.105 + ] + }, + "choice_is_modal": true, + "uiux_product_n": 54, + "uiux_product_tune": { + "k": 7, + "n": 54, + "p": 0.13, + "ci95": [ + 0.064, + 0.244 + ] + }, + "uiux_product_tune_primary_or_secondary": { + "k": 24, + "n": 54, + "p": 0.444, + "ci95": [ + 0.32, + 0.576 + ] + }, + "tune_primary_or_secondary": { + "k": 29, + "n": 132, + "p": 0.22, + "ci95": [ + 0.158, + 0.298 + ] + }, + "check_primary_or_secondary": { + "k": 43, + "n": 132, + "p": 0.326, + "ci95": [ + 0.252, + 0.41 + ] + }, + "explore_primary_or_secondary": { + "k": 43, + "n": 132, + "p": 0.326, + "ci95": [ + 0.252, + 0.41 + ] + } + }, + "documents": { + "choice": { + "k": 31, + "n": 41, + "p": 0.756, + "ci95": [ + 0.607, + 0.862 + ] + }, + "tune": { + "k": 0, + "n": 41, + "p": 0.0, + "ci95": [ + 0, + 0.086 + ] + }, + "choice_is_modal": true, + "uiux_product_n": 0, + "uiux_product_tune": { + "k": 0, + "n": 0, + "p": null, + "ci95": null + }, + "uiux_product_tune_primary_or_secondary": { + "k": 0, + "n": 0, + "p": null, + "ci95": null + }, + "tune_primary_or_secondary": { + "k": 2, + "n": 41, + "p": 0.049, + "ci95": [ + 0.013, + 0.161 + ] + }, + "check_primary_or_secondary": { + "k": 12, + "n": 41, + "p": 0.293, + "ci95": [ + 0.176, + 0.445 + ] + }, + "explore_primary_or_secondary": { + "k": 8, + "n": 41, + "p": 0.195, + "ci95": [ + 0.102, + 0.34 + ] + } + }, + "small": { + "choice": { + "k": 8, + "n": 11, + "p": 0.727, + "ci95": [ + 0.434, + 0.903 + ] + }, + "tune": { + "k": 1, + "n": 11, + "p": 0.091, + "ci95": [ + 0.016, + 0.377 + ] + }, + "choice_is_modal": true, + "uiux_product_n": 6, + "uiux_product_tune": { + "k": 1, + "n": 6, + "p": 0.167, + "ci95": [ + 0.03, + 0.564 + ] + }, + "uiux_product_tune_primary_or_secondary": { + "k": 2, + "n": 6, + "p": 0.333, + "ci95": [ + 0.097, + 0.7 + ] + }, + "tune_primary_or_secondary": { + "k": 3, + "n": 11, + "p": 0.273, + "ci95": [ + 0.097, + 0.566 + ] + }, + "check_primary_or_secondary": { + "k": 4, + "n": 11, + "p": 0.364, + "ci95": [ + 0.152, + 0.646 + ] + }, + "explore_primary_or_secondary": { + "k": 4, + "n": 11, + "p": 0.364, + "ci95": [ + 0.152, + 0.646 + ] + } + } + }, + "H3": { + "all": { + "n_choice": 124, + "options_median": 5.0, + "options_ge4": { + "k": 110, + "n": 124, + "p": 0.887, + "ci95": [ + 0.819, + 0.932 + ] + }, + "criteria_nonempty": { + "k": 124, + "n": 124, + "p": 1.0, + "ci95": [ + 0.97, + 1.0 + ] + }, + "composite_answer": { + "k": 68, + "n": 124, + "p": 0.548, + "ci95": [ + 0.461, + 0.633 + ] + }, + "has_deferred_option": { + "k": 69, + "n": 124, + "p": 0.556, + "ci95": [ + 0.469, + 0.641 + ] + } + }, + "threads": { + "n_choice": 93, + "options_median": 6, + "options_ge4": { + "k": 91, + "n": 93, + "p": 0.978, + "ci95": [ + 0.925, + 0.994 + ] + }, + "criteria_nonempty": { + "k": 93, + "n": 93, + "p": 1.0, + "ci95": [ + 0.96, + 1.0 + ] + }, + "composite_answer": { + "k": 52, + "n": 93, + "p": 0.559, + "ci95": [ + 0.458, + 0.656 + ] + }, + "has_deferred_option": { + "k": 59, + "n": 93, + "p": 0.634, + "ci95": [ + 0.533, + 0.725 + ] + } + }, + "documents": { + "n_choice": 31, + "options_median": 4, + "options_ge4": { + "k": 19, + "n": 31, + "p": 0.613, + "ci95": [ + 0.438, + 0.763 + ] + }, + "criteria_nonempty": { + "k": 31, + "n": 31, + "p": 1.0, + "ci95": [ + 0.89, + 1.0 + ] + }, + "composite_answer": { + "k": 16, + "n": 31, + "p": 0.516, + "ci95": [ + 0.348, + 0.68 + ] + }, + "has_deferred_option": { + "k": 10, + "n": 31, + "p": 0.323, + "ci95": [ + 0.186, + 0.499 + ] + } + }, + "small": { + "n_choice": 8, + "options_median": 5.5, + "options_ge4": { + "k": 7, + "n": 8, + "p": 0.875, + "ci95": [ + 0.529, + 0.978 + ] + }, + "criteria_nonempty": { + "k": 8, + "n": 8, + "p": 1.0, + "ci95": [ + 0.676, + 1 + ] + }, + "composite_answer": { + "k": 6, + "n": 8, + "p": 0.75, + "ci95": [ + 0.409, + 0.929 + ] + }, + "has_deferred_option": { + "k": 4, + "n": 8, + "p": 0.5, + "ci95": [ + 0.215, + 0.785 + ] + } + } + }, + "H4": { + "all": { + "order_guaranteed": false, + "first_option_chosen": { + "k": 57, + "n": 148, + "p": 0.385, + "ci95": [ + 0.311, + 0.465 + ] + }, + "first_option_chosen_by_source": { + "thread": { + "k": 42, + "n": 117, + "p": 0.359, + "ci95": [ + 0.278, + 0.449 + ] + }, + "document": { + "k": 15, + "n": 31, + "p": 0.484, + "ci95": [ + 0.32, + 0.652 + ] + } + } + }, + "threads": { + "order_guaranteed": false, + "first_option_chosen": { + "k": 42, + "n": 117, + "p": 0.359, + "ci95": [ + 0.278, + 0.449 + ] + }, + "first_option_chosen_by_source": { + "thread": { + "k": 42, + "n": 117, + "p": 0.359, + "ci95": [ + 0.278, + 0.449 + ] + }, + "document": { + "k": 0, + "n": 0, + "p": null, + "ci95": null + } + } + }, + "documents": { + "order_guaranteed": false, + "first_option_chosen": { + "k": 15, + "n": 31, + "p": 0.484, + "ci95": [ + 0.32, + 0.652 + ] + }, + "first_option_chosen_by_source": { + "thread": { + "k": 0, + "n": 0, + "p": null, + "ci95": null + }, + "document": { + "k": 15, + "n": 31, + "p": 0.484, + "ci95": [ + 0.32, + 0.652 + ] + } + } + }, + "small": { + "order_guaranteed": false, + "first_option_chosen": { + "k": 5, + "n": 10, + "p": 0.5, + "ci95": [ + 0.237, + 0.763 + ] + }, + "first_option_chosen_by_source": { + "thread": { + "k": 5, + "n": 10, + "p": 0.5, + "ci95": [ + 0.237, + 0.763 + ] + }, + "document": { + "k": 0, + "n": 0, + "p": null, + "ci95": null + } + } + } + }, + "H5": { + "all": { + "deferred_flag": { + "k": 66, + "n": 173, + "p": 0.382, + "ci95": [ + 0.312, + 0.456 + ] + }, + "undecided_and_deferred": { + "k": 23, + "n": 173, + "p": 0.133, + "ci95": [ + 0.09, + 0.192 + ] + }, + "parked_any": { + "k": 42, + "n": 173, + "p": 0.243, + "ci95": [ + 0.185, + 0.312 + ] + }, + "undecided_n": 25, + "undecided_deferred": 23, + "undecided_abandoned": 3, + "decided_but_deferred_flag": 43 + }, + "threads": { + "deferred_flag": { + "k": 47, + "n": 132, + "p": 0.356, + "ci95": [ + 0.28, + 0.441 + ] + }, + "undecided_and_deferred": { + "k": 13, + "n": 132, + "p": 0.098, + "ci95": [ + 0.058, + 0.161 + ] + }, + "parked_any": { + "k": 30, + "n": 132, + "p": 0.227, + "ci95": [ + 0.164, + 0.306 + ] + }, + "undecided_n": 15, + "undecided_deferred": 13, + "undecided_abandoned": 3, + "decided_but_deferred_flag": 34 + }, + "documents": { + "deferred_flag": { + "k": 19, + "n": 41, + "p": 0.463, + "ci95": [ + 0.321, + 0.613 + ] + }, + "undecided_and_deferred": { + "k": 10, + "n": 41, + "p": 0.244, + "ci95": [ + 0.138, + 0.393 + ] + }, + "parked_any": { + "k": 12, + "n": 41, + "p": 0.293, + "ci95": [ + 0.176, + 0.445 + ] + }, + "undecided_n": 10, + "undecided_deferred": 10, + "undecided_abandoned": 0, + "decided_but_deferred_flag": 9 + }, + "small": { + "deferred_flag": { + "k": 3, + "n": 11, + "p": 0.273, + "ci95": [ + 0.097, + 0.566 + ] + }, + "undecided_and_deferred": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "parked_any": { + "k": 4, + "n": 11, + "p": 0.364, + "ci95": [ + 0.152, + 0.646 + ] + }, + "undecided_n": 1, + "undecided_deferred": 0, + "undecided_abandoned": 0, + "decided_but_deferred_flag": 3 + } + }, + "H6": { + "all": { + "subq_nonempty": { + "k": 171, + "n": 173, + "p": 0.988, + "ci95": [ + 0.959, + 0.997 + ] + }, + "struct_tag": { + "k": 113, + "n": 173, + "p": 0.653, + "ci95": [ + 0.58, + 0.72 + ] + }, + "rfc_or_proposal_n": 31, + "rfc_or_proposal_struct_tag": { + "k": 26, + "n": 31, + "p": 0.839, + "ci95": [ + 0.674, + 0.929 + ] + }, + "follow_up_structure": { + "k": 87, + "n": 173, + "p": 0.503, + "ci95": [ + 0.429, + 0.577 + ] + } + }, + "threads": { + "subq_nonempty": { + "k": 130, + "n": 132, + "p": 0.985, + "ci95": [ + 0.946, + 0.996 + ] + }, + "struct_tag": { + "k": 81, + "n": 132, + "p": 0.614, + "ci95": [ + 0.528, + 0.692 + ] + }, + "rfc_or_proposal_n": 22, + "rfc_or_proposal_struct_tag": { + "k": 18, + "n": 22, + "p": 0.818, + "ci95": [ + 0.615, + 0.927 + ] + }, + "follow_up_structure": { + "k": 59, + "n": 132, + "p": 0.447, + "ci95": [ + 0.365, + 0.532 + ] + } + }, + "documents": { + "subq_nonempty": { + "k": 41, + "n": 41, + "p": 1.0, + "ci95": [ + 0.914, + 1 + ] + }, + "struct_tag": { + "k": 32, + "n": 41, + "p": 0.78, + "ci95": [ + 0.633, + 0.88 + ] + }, + "rfc_or_proposal_n": 9, + "rfc_or_proposal_struct_tag": { + "k": 8, + "n": 9, + "p": 0.889, + "ci95": [ + 0.565, + 0.98 + ] + }, + "follow_up_structure": { + "k": 28, + "n": 41, + "p": 0.683, + "ci95": [ + 0.53, + 0.804 + ] + } + }, + "small": { + "subq_nonempty": { + "k": 11, + "n": 11, + "p": 1.0, + "ci95": [ + 0.741, + 1 + ] + }, + "struct_tag": { + "k": 5, + "n": 11, + "p": 0.455, + "ci95": [ + 0.213, + 0.72 + ] + }, + "rfc_or_proposal_n": 0, + "rfc_or_proposal_struct_tag": { + "k": 0, + "n": 0, + "p": null, + "ci95": null + }, + "follow_up_structure": { + "k": 5, + "n": 11, + "p": 0.455, + "ci95": [ + 0.213, + 0.72 + ] + } + } + }, + "H7": { + "all": { + "n_noncheck": 161, + "research_or_spike": { + "k": 142, + "n": 161, + "p": 0.882, + "ci95": [ + 0.823, + 0.923 + ] + }, + "strict_(research_beyond_links_or_spike)": { + "k": 112, + "n": 161, + "p": 0.696, + "ci95": [ + 0.621, + 0.761 + ] + }, + "spike_any": { + "k": 108, + "n": 161, + "p": 0.671, + "ci95": [ + 0.595, + 0.739 + ] + }, + "strict_excluding_implementation_pr": { + "k": 56, + "n": 161, + "p": 0.348, + "ci95": [ + 0.279, + 0.424 + ] + }, + "spike_dist": { + "implementation-pr": 64, + "none": 53, + "prototype": 35, + "feature-flag": 5, + "experiment": 4 + }, + "evidence_tag": { + "k": 43, + "n": 161, + "p": 0.267, + "ci95": [ + 0.205, + 0.34 + ] + }, + "check_secondary": { + "k": 43, + "n": 161, + "p": 0.267, + "ci95": [ + 0.205, + 0.34 + ] + } + }, + "threads": { + "n_noncheck": 121, + "research_or_spike": { + "k": 111, + "n": 121, + "p": 0.917, + "ci95": [ + 0.855, + 0.954 + ] + }, + "strict_(research_beyond_links_or_spike)": { + "k": 96, + "n": 121, + "p": 0.793, + "ci95": [ + 0.713, + 0.856 + ] + }, + "spike_any": { + "k": 92, + "n": 121, + "p": 0.76, + "ci95": [ + 0.677, + 0.828 + ] + }, + "strict_excluding_implementation_pr": { + "k": 42, + "n": 121, + "p": 0.347, + "ci95": [ + 0.268, + 0.435 + ] + }, + "spike_dist": { + "implementation-pr": 59, + "none": 29, + "prototype": 28, + "feature-flag": 4, + "experiment": 1 + }, + "evidence_tag": { + "k": 34, + "n": 121, + "p": 0.281, + "ci95": [ + 0.209, + 0.367 + ] + }, + "check_secondary": { + "k": 32, + "n": 121, + "p": 0.264, + "ci95": [ + 0.194, + 0.349 + ] + } + }, + "documents": { + "n_noncheck": 40, + "research_or_spike": { + "k": 31, + "n": 40, + "p": 0.775, + "ci95": [ + 0.625, + 0.877 + ] + }, + "strict_(research_beyond_links_or_spike)": { + "k": 16, + "n": 40, + "p": 0.4, + "ci95": [ + 0.263, + 0.554 + ] + }, + "spike_any": { + "k": 16, + "n": 40, + "p": 0.4, + "ci95": [ + 0.263, + 0.554 + ] + }, + "strict_excluding_implementation_pr": { + "k": 14, + "n": 40, + "p": 0.35, + "ci95": [ + 0.221, + 0.505 + ] + }, + "spike_dist": { + "none": 24, + "prototype": 7, + "implementation-pr": 5, + "experiment": 3, + "feature-flag": 1 + }, + "evidence_tag": { + "k": 9, + "n": 40, + "p": 0.225, + "ci95": [ + 0.123, + 0.375 + ] + }, + "check_secondary": { + "k": 11, + "n": 40, + "p": 0.275, + "ci95": [ + 0.161, + 0.428 + ] + } + }, + "small": { + "n_noncheck": 10, + "research_or_spike": { + "k": 10, + "n": 10, + "p": 1.0, + "ci95": [ + 0.722, + 1 + ] + }, + "strict_(research_beyond_links_or_spike)": { + "k": 9, + "n": 10, + "p": 0.9, + "ci95": [ + 0.596, + 0.982 + ] + }, + "spike_any": { + "k": 9, + "n": 10, + "p": 0.9, + "ci95": [ + 0.596, + 0.982 + ] + }, + "strict_excluding_implementation_pr": { + "k": 4, + "n": 10, + "p": 0.4, + "ci95": [ + 0.168, + 0.687 + ] + }, + "spike_dist": { + "implementation-pr": 5, + "prototype": 4, + "none": 1 + }, + "evidence_tag": { + "k": 3, + "n": 10, + "p": 0.3, + "ci95": [ + 0.108, + 0.603 + ] + }, + "check_secondary": { + "k": 3, + "n": 10, + "p": 0.3, + "ci95": [ + 0.108, + 0.603 + ] + } + } + }, + "H8": { + "all": { + "n_decided": 148, + "reversibility_stated": { + "k": 60, + "n": 148, + "p": 0.405, + "ci95": [ + 0.33, + 0.486 + ] + }, + "confidence_stated": { + "k": 99, + "n": 148, + "p": 0.669, + "ci95": [ + 0.59, + 0.74 + ] + }, + "confidence_stated_by_source": { + "thread": { + "k": 85, + "n": 117, + "p": 0.726, + "ci95": [ + 0.639, + 0.799 + ] + }, + "document": { + "k": 14, + "n": 31, + "p": 0.452, + "ci95": [ + 0.292, + 0.622 + ] + } + }, + "provisional_or_conditional_tag": { + "k": 24, + "n": 173, + "p": 0.139, + "ci95": [ + 0.095, + 0.198 + ] + } + }, + "threads": { + "n_decided": 117, + "reversibility_stated": { + "k": 51, + "n": 117, + "p": 0.436, + "ci95": [ + 0.349, + 0.526 + ] + }, + "confidence_stated": { + "k": 85, + "n": 117, + "p": 0.726, + "ci95": [ + 0.639, + 0.799 + ] + }, + "confidence_stated_by_source": { + "thread": { + "k": 85, + "n": 117, + "p": 0.726, + "ci95": [ + 0.639, + 0.799 + ] + }, + "document": { + "k": 0, + "n": 0, + "p": null, + "ci95": null + } + }, + "provisional_or_conditional_tag": { + "k": 10, + "n": 132, + "p": 0.076, + "ci95": [ + 0.042, + 0.134 + ] + } + }, + "documents": { + "n_decided": 31, + "reversibility_stated": { + "k": 9, + "n": 31, + "p": 0.29, + "ci95": [ + 0.161, + 0.466 + ] + }, + "confidence_stated": { + "k": 14, + "n": 31, + "p": 0.452, + "ci95": [ + 0.292, + 0.622 + ] + }, + "confidence_stated_by_source": { + "thread": { + "k": 0, + "n": 0, + "p": null, + "ci95": null + }, + "document": { + "k": 14, + "n": 31, + "p": 0.452, + "ci95": [ + 0.292, + 0.622 + ] + } + }, + "provisional_or_conditional_tag": { + "k": 14, + "n": 41, + "p": 0.341, + "ci95": [ + 0.216, + 0.495 + ] + } + }, + "small": { + "n_decided": 10, + "reversibility_stated": { + "k": 2, + "n": 10, + "p": 0.2, + "ci95": [ + 0.057, + 0.51 + ] + }, + "confidence_stated": { + "k": 5, + "n": 10, + "p": 0.5, + "ci95": [ + 0.237, + 0.763 + ] + }, + "confidence_stated_by_source": { + "thread": { + "k": 5, + "n": 10, + "p": 0.5, + "ci95": [ + 0.237, + 0.763 + ] + }, + "document": { + "k": 0, + "n": 0, + "p": null, + "ci95": null + } + }, + "provisional_or_conditional_tag": { + "k": 1, + "n": 11, + "p": 0.091, + "ci95": [ + 0.016, + 0.377 + ] + } + } + }, + "H9": { + "all": { + "decided_ge3_n": 132, + "dissent_decided_ge3": { + "k": 116, + "n": 132, + "p": 0.879, + "ci95": [ + 0.812, + 0.924 + ] + }, + "dissent_all_decided": { + "k": 118, + "n": 148, + "p": 0.797, + "ci95": [ + 0.725, + 0.854 + ] + } + }, + "threads": { + "decided_ge3_n": 117, + "dissent_decided_ge3": { + "k": 104, + "n": 117, + "p": 0.889, + "ci95": [ + 0.819, + 0.934 + ] + }, + "dissent_all_decided": { + "k": 104, + "n": 117, + "p": 0.889, + "ci95": [ + 0.819, + 0.934 + ] + } + }, + "documents": { + "decided_ge3_n": 15, + "dissent_decided_ge3": { + "k": 12, + "n": 15, + "p": 0.8, + "ci95": [ + 0.548, + 0.93 + ] + }, + "dissent_all_decided": { + "k": 14, + "n": 31, + "p": 0.452, + "ci95": [ + 0.292, + 0.622 + ] + } + }, + "small": { + "decided_ge3_n": 10, + "dissent_decided_ge3": { + "k": 6, + "n": 10, + "p": 0.6, + "ci95": [ + 0.313, + 0.832 + ] + }, + "dissent_all_decided": { + "k": 6, + "n": 10, + "p": 0.6, + "ci95": [ + 0.313, + 0.832 + ] + } + } + }, + "H10": { + "all": { + "reopened_or_superseded": { + "k": 50, + "n": 173, + "p": 0.289, + "ci95": [ + 0.227, + 0.361 + ] + }, + "reopened": { + "k": 33, + "n": 173, + "p": 0.191, + "ci95": [ + 0.139, + 0.256 + ] + }, + "superseded": { + "k": 25, + "n": 173, + "p": 0.145, + "ci95": [ + 0.1, + 0.205 + ] + }, + "of_those_supersede_or_revert_tag": { + "k": 35, + "n": 50, + "p": 0.7, + "ci95": [ + 0.562, + 0.809 + ] + }, + "supersede_or_revert_tag_any": { + "k": 55, + "n": 173, + "p": 0.318, + "ci95": [ + 0.253, + 0.391 + ] + } + }, + "threads": { + "reopened_or_superseded": { + "k": 35, + "n": 132, + "p": 0.265, + "ci95": [ + 0.197, + 0.346 + ] + }, + "reopened": { + "k": 19, + "n": 132, + "p": 0.144, + "ci95": [ + 0.094, + 0.214 + ] + }, + "superseded": { + "k": 16, + "n": 132, + "p": 0.121, + "ci95": [ + 0.076, + 0.188 + ] + }, + "of_those_supersede_or_revert_tag": { + "k": 26, + "n": 35, + "p": 0.743, + "ci95": [ + 0.579, + 0.858 + ] + }, + "supersede_or_revert_tag_any": { + "k": 43, + "n": 132, + "p": 0.326, + "ci95": [ + 0.252, + 0.41 + ] + } + }, + "documents": { + "reopened_or_superseded": { + "k": 15, + "n": 41, + "p": 0.366, + "ci95": [ + 0.236, + 0.519 + ] + }, + "reopened": { + "k": 14, + "n": 41, + "p": 0.341, + "ci95": [ + 0.216, + 0.495 + ] + }, + "superseded": { + "k": 9, + "n": 41, + "p": 0.22, + "ci95": [ + 0.12, + 0.367 + ] + }, + "of_those_supersede_or_revert_tag": { + "k": 9, + "n": 15, + "p": 0.6, + "ci95": [ + 0.357, + 0.802 + ] + }, + "supersede_or_revert_tag_any": { + "k": 12, + "n": 41, + "p": 0.293, + "ci95": [ + 0.176, + 0.445 + ] + } + }, + "small": { + "reopened_or_superseded": { + "k": 5, + "n": 11, + "p": 0.455, + "ci95": [ + 0.213, + 0.72 + ] + }, + "reopened": { + "k": 5, + "n": 11, + "p": 0.455, + "ci95": [ + 0.213, + 0.72 + ] + }, + "superseded": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "of_those_supersede_or_revert_tag": { + "k": 5, + "n": 5, + "p": 1.0, + "ci95": [ + 0.566, + 1 + ] + }, + "supersede_or_revert_tag_any": { + "k": 5, + "n": 11, + "p": 0.455, + "ci95": [ + 0.213, + 0.72 + ] + } + } + }, + "H11": { + "all": { + "pr_source_and_impl_pr": { + "k": 29, + "n": 173, + "p": 0.168, + "ci95": [ + 0.119, + 0.23 + ] + }, + "impl_pr_any_source": { + "k": 71, + "n": 173, + "p": 0.41, + "ci95": [ + 0.34, + 0.485 + ] + }, + "pr_source": { + "k": 34, + "n": 173, + "p": 0.197, + "ci95": [ + 0.144, + 0.262 + ] + }, + "implicit_tag": { + "k": 22, + "n": 173, + "p": 0.127, + "ci95": [ + 0.085, + 0.185 + ] + }, + "implementedIn_nonempty": { + "k": 76, + "n": 173, + "p": 0.439, + "ci95": [ + 0.367, + 0.514 + ] + } + }, + "threads": { + "pr_source_and_impl_pr": { + "k": 29, + "n": 132, + "p": 0.22, + "ci95": [ + 0.158, + 0.298 + ] + }, + "impl_pr_any_source": { + "k": 66, + "n": 132, + "p": 0.5, + "ci95": [ + 0.416, + 0.584 + ] + }, + "pr_source": { + "k": 34, + "n": 132, + "p": 0.258, + "ci95": [ + 0.191, + 0.338 + ] + }, + "implicit_tag": { + "k": 19, + "n": 132, + "p": 0.144, + "ci95": [ + 0.094, + 0.214 + ] + }, + "implementedIn_nonempty": { + "k": 73, + "n": 132, + "p": 0.553, + "ci95": [ + 0.468, + 0.635 + ] + } + }, + "documents": { + "pr_source_and_impl_pr": { + "k": 0, + "n": 41, + "p": 0.0, + "ci95": [ + 0, + 0.086 + ] + }, + "impl_pr_any_source": { + "k": 5, + "n": 41, + "p": 0.122, + "ci95": [ + 0.053, + 0.255 + ] + }, + "pr_source": { + "k": 0, + "n": 41, + "p": 0.0, + "ci95": [ + 0, + 0.086 + ] + }, + "implicit_tag": { + "k": 3, + "n": 41, + "p": 0.073, + "ci95": [ + 0.025, + 0.194 + ] + }, + "implementedIn_nonempty": { + "k": 3, + "n": 41, + "p": 0.073, + "ci95": [ + 0.025, + 0.194 + ] + } + }, + "small": { + "pr_source_and_impl_pr": { + "k": 1, + "n": 11, + "p": 0.091, + "ci95": [ + 0.016, + 0.377 + ] + }, + "impl_pr_any_source": { + "k": 5, + "n": 11, + "p": 0.455, + "ci95": [ + 0.213, + 0.72 + ] + }, + "pr_source": { + "k": 1, + "n": 11, + "p": 0.091, + "ci95": [ + 0.016, + 0.377 + ] + }, + "implicit_tag": { + "k": 1, + "n": 11, + "p": 0.091, + "ci95": [ + 0.016, + 0.377 + ] + }, + "implementedIn_nonempty": { + "k": 8, + "n": 11, + "p": 0.727, + "ci95": [ + 0.434, + 0.903 + ] + } + } + }, + "H12": { + "all": { + "le3_n": 3, + "le3_ids": [ + "excalidraw-embed-validator-api", + "excalidraw-embed-alt-text", + "vscode-fixed-width-tabs" + ], + "le3_single_decider": { + "k": 2, + "n": 3, + "p": 0.667, + "ci95": [ + 0.208, + 0.939 + ] + }, + "single_decider": { + "k": 57, + "n": 173, + "p": 0.329, + "ci95": [ + 0.264, + 0.403 + ] + }, + "maintainer_decision_how": { + "k": 83, + "n": 173, + "p": 0.48, + "ci95": [ + 0.407, + 0.554 + ] + }, + "team_or_consensus_decider": { + "k": 89, + "n": 173, + "p": 0.514, + "ci95": [ + 0.44, + 0.588 + ] + }, + "decider_unclear": { + "k": 21, + "n": 173, + "p": 0.121, + "ci95": [ + 0.081, + 0.178 + ] + }, + "vote_or_poll_nonstandards": { + "k": 15, + "n": 157, + "p": 0.096, + "ci95": [ + 0.059, + 0.152 + ] + }, + "vote_or_poll_nonformal": { + "k": 8, + "n": 122, + "p": 0.066, + "ci95": [ + 0.034, + 0.124 + ] + }, + "vote_or_poll_all": { + "k": 17, + "n": 173, + "p": 0.098, + "ci95": [ + 0.062, + 0.152 + ] + } + }, + "threads": { + "le3_n": 3, + "le3_ids": [ + "excalidraw-embed-validator-api", + "excalidraw-embed-alt-text", + "vscode-fixed-width-tabs" + ], + "le3_single_decider": { + "k": 2, + "n": 3, + "p": 0.667, + "ci95": [ + 0.208, + 0.939 + ] + }, + "single_decider": { + "k": 54, + "n": 132, + "p": 0.409, + "ci95": [ + 0.329, + 0.494 + ] + }, + "maintainer_decision_how": { + "k": 78, + "n": 132, + "p": 0.591, + "ci95": [ + 0.506, + 0.671 + ] + }, + "team_or_consensus_decider": { + "k": 60, + "n": 132, + "p": 0.455, + "ci95": [ + 0.372, + 0.54 + ] + }, + "decider_unclear": { + "k": 15, + "n": 132, + "p": 0.114, + "ci95": [ + 0.07, + 0.179 + ] + }, + "vote_or_poll_nonstandards": { + "k": 11, + "n": 131, + "p": 0.084, + "ci95": [ + 0.048, + 0.144 + ] + }, + "vote_or_poll_nonformal": { + "k": 7, + "n": 106, + "p": 0.066, + "ci95": [ + 0.032, + 0.13 + ] + }, + "vote_or_poll_all": { + "k": 11, + "n": 132, + "p": 0.083, + "ci95": [ + 0.047, + 0.143 + ] + } + }, + "documents": { + "le3_n": 0, + "le3_ids": [], + "le3_single_decider": { + "k": 0, + "n": 0, + "p": null, + "ci95": null + }, + "single_decider": { + "k": 3, + "n": 41, + "p": 0.073, + "ci95": [ + 0.025, + 0.194 + ] + }, + "maintainer_decision_how": { + "k": 5, + "n": 41, + "p": 0.122, + "ci95": [ + 0.053, + 0.255 + ] + }, + "team_or_consensus_decider": { + "k": 29, + "n": 41, + "p": 0.707, + "ci95": [ + 0.555, + 0.824 + ] + }, + "decider_unclear": { + "k": 6, + "n": 41, + "p": 0.146, + "ci95": [ + 0.069, + 0.284 + ] + }, + "vote_or_poll_nonstandards": { + "k": 4, + "n": 26, + "p": 0.154, + "ci95": [ + 0.061, + 0.335 + ] + }, + "vote_or_poll_nonformal": { + "k": 1, + "n": 16, + "p": 0.062, + "ci95": [ + 0.011, + 0.283 + ] + }, + "vote_or_poll_all": { + "k": 6, + "n": 41, + "p": 0.146, + "ci95": [ + 0.069, + 0.284 + ] + } + }, + "small": { + "le3_n": 0, + "le3_ids": [], + "le3_single_decider": { + "k": 0, + "n": 0, + "p": null, + "ci95": null + }, + "single_decider": { + "k": 9, + "n": 11, + "p": 0.818, + "ci95": [ + 0.523, + 0.949 + ] + }, + "maintainer_decision_how": { + "k": 10, + "n": 11, + "p": 0.909, + "ci95": [ + 0.623, + 0.984 + ] + }, + "team_or_consensus_decider": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "decider_unclear": { + "k": 2, + "n": 11, + "p": 0.182, + "ci95": [ + 0.051, + 0.477 + ] + }, + "vote_or_poll_nonstandards": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "vote_or_poll_nonformal": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "vote_or_poll_all": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + } + } + } + }, + "tags": { + "venue-split": { + "all": { + "k": 78, + "n": 173, + "p": 0.451, + "ci95": [ + 0.379, + 0.525 + ] + }, + "threads": { + "k": 61, + "n": 132, + "p": 0.462, + "ci95": [ + 0.379, + 0.547 + ] + }, + "documents": { + "k": 17, + "n": 41, + "p": 0.415, + "ci95": [ + 0.278, + 0.566 + ] + }, + "small": { + "k": 1, + "n": 11, + "p": 0.091, + "ci95": [ + 0.016, + 0.377 + ] + }, + "entries_total": 84, + "formal_share_of_hits": { + "k": 25, + "n": 78, + "p": 0.321, + "ci95": [ + 0.227, + 0.43 + ] + }, + "examples": [ + { + "id": "bat-builtin-pager-approach", + "source": "thread", + "url": "https://github.com/sharkdp/bat/issues/1053", + "gap": "off-platform decision: the choice of minus was made in PR #3402 and only noticed here afterwards" + }, + { + "id": "astro-scripts-only-when-rendered", + "source": "thread", + "url": "https://github.com/withastro/roadmap/discussions/664", + "gap": "off-platform decision: early objections came from Discord and are only paraphrased in the thread" + }, + { + "id": "bat-debian-batcat-name", + "source": "thread", + "url": "https://github.com/sharkdp/bat/issues/2455", + "gap": "off-platform decision: the chosen path depends on a rename request filed on Bacula's GitLab, outside the thread" + }, + { + "id": "cargo-feature-doc-field", + "source": "thread", + "url": "https://github.com/rust-lang/rfcs/pull/3485", + "gap": "off-platform decision: blocking concerns were cleared in a Cargo team meeting and an unconference, only reported here" + } + ] + }, + "staged": { + "all": { + "k": 55, + "n": 173, + "p": 0.318, + "ci95": [ + 0.253, + 0.391 + ] + }, + "threads": { + "k": 44, + "n": 132, + "p": 0.333, + "ci95": [ + 0.259, + 0.417 + ] + }, + "documents": { + "k": 11, + "n": 41, + "p": 0.268, + "ci95": [ + 0.157, + 0.419 + ] + }, + "small": { + "k": 3, + "n": 11, + "p": 0.273, + "ci95": [ + 0.097, + 0.566 + ] + }, + "entries_total": 56, + "formal_share_of_hits": { + "k": 21, + "n": 55, + "p": 0.382, + "ci95": [ + 0.265, + 0.514 + ] + }, + "examples": [ + { + "id": "bat-builtin-pager-approach", + "source": "thread", + "url": "https://github.com/sharkdp/bat/issues/1053", + "gap": "staged acceptance: the pager shipped opt-in with known gaps while the issue stays open as the umbrella" + }, + { + "id": "delta-light-dark-detection", + "source": "thread", + "url": "https://github.com/dandavison/delta/issues/447", + "gap": "community workarounds as interim answers: many user snippets served as de facto solutions for years without being recorded as partial outcomes" + }, + { + "id": "just-parallel-mechanism", + "source": "thread", + "url": "https://github.com/casey/just/issues/626", + "gap": "staged acceptance: the attribute shipped while the CLI flag and job cap were explicitly deferred to new issues" + }, + { + "id": "astro-define-vars-scripts", + "source": "thread", + "url": "https://github.com/withastro/roadmap/discussions/1246", + "gap": "staged acceptance: Stage 2 approves the direction while deprecation and removal span future majors" + } + ] + }, + "other:misc": { + "all": { + "k": 52, + "n": 173, + "p": 0.301, + "ci95": [ + 0.237, + 0.373 + ] + }, + "threads": { + "k": 42, + "n": 132, + "p": 0.318, + "ci95": [ + 0.245, + 0.402 + ] + }, + "documents": { + "k": 10, + "n": 41, + "p": 0.244, + "ci95": [ + 0.138, + 0.393 + ] + }, + "small": { + "k": 5, + "n": 11, + "p": 0.455, + "ci95": [ + 0.213, + 0.72 + ] + }, + "entries_total": 56, + "formal_share_of_hits": { + "k": 16, + "n": 52, + "p": 0.308, + "ci95": [ + 0.199, + 0.443 + ] + }, + "examples": [ + { + "id": "fzf-default-opts-file", + "source": "thread", + "url": "https://github.com/junegunn/fzf/discussions/2528", + "gap": "answer findability: later users re-asked because the settled answer was buried in a long open discussion that was never marked answered" + }, + { + "id": "fzf-preview-window", + "source": "thread", + "url": "https://github.com/junegunn/fzf/issues/587", + "gap": "support noise mixed in: an unrelated install problem interleaves with the design discussion" + }, + { + "id": "ripgrep-long-line-handling", + "source": "thread", + "url": "https://github.com/BurntSushi/ripgrep/issues/129", + "gap": "spec iteration in comments: the proposal was restated four times as edited specs, with no versioned draft" + }, + { + "id": "ripgrep-pager-flag", + "source": "thread", + "url": "https://github.com/BurntSushi/ripgrep/issues/86", + "gap": "duplicates merged: a fresh duplicate request elsewhere triggered the reopen" + } + ] + }, + "coupled": { + "all": { + "k": 51, + "n": 173, + "p": 0.295, + "ci95": [ + 0.232, + 0.367 + ] + }, + "threads": { + "k": 43, + "n": 132, + "p": 0.326, + "ci95": [ + 0.252, + 0.41 + ] + }, + "documents": { + "k": 8, + "n": 41, + "p": 0.195, + "ci95": [ + 0.102, + 0.34 + ] + }, + "small": { + "k": 2, + "n": 11, + "p": 0.182, + "ci95": [ + 0.051, + 0.477 + ] + }, + "entries_total": 51, + "formal_share_of_hits": { + "k": 16, + "n": 51, + "p": 0.314, + "ci95": [ + 0.203, + 0.45 + ] + }, + "examples": [ + { + "id": "just-f-string-syntax", + "source": "thread", + "url": "https://github.com/casey/just/issues/11", + "gap": "coupled decisions: the brace choice was tied to relaxing a separate rule in recipe interpolations" + }, + { + "id": "ripgrep-long-line-handling", + "source": "thread", + "url": "https://github.com/BurntSushi/ripgrep/issues/129", + "gap": "coupled decisions: default-on behaviour, units and terminal detection were settled together" + }, + { + "id": "appflowy-dark-mode-palette", + "source": "thread", + "url": "https://github.com/AppFlowy-IO/AppFlowy/issues/31", + "gap": "coupled decisions: palette choice, toggle-versus-theming scope and colour-token renaming were settled together in one thread" + }, + { + "id": "astro-define-vars-scripts", + "source": "thread", + "url": "https://github.com/withastro/roadmap/discussions/1246", + "gap": "coupled decisions: the outcome is tied to the separate inline-script proposal #995" + } + ] + }, + "evidence-not-check": { + "all": { + "k": 49, + "n": 173, + "p": 0.283, + "ci95": [ + 0.221, + 0.355 + ] + }, + "threads": { + "k": 40, + "n": 132, + "p": 0.303, + "ci95": [ + 0.231, + 0.386 + ] + }, + "documents": { + "k": 9, + "n": 41, + "p": 0.22, + "ci95": [ + 0.12, + 0.367 + ] + }, + "small": { + "k": 3, + "n": 11, + "p": 0.273, + "ci95": [ + 0.097, + 0.566 + ] + }, + "entries_total": 53, + "formal_share_of_hits": { + "k": 10, + "n": 49, + "p": 0.204, + "ci95": [ + 0.115, + 0.336 + ] + }, + "examples": [ + { + "id": "just-parallel-mechanism", + "source": "thread", + "url": "https://github.com/casey/just/issues/626", + "gap": "competing implementations: a community draft PR implementing both options went unreviewed while the maintainer later shipped a different, narrower PR" + }, + { + "id": "ripgrep-hyperlink-support", + "source": "thread", + "url": "https://github.com/BurntSushi/ripgrep/issues/665", + "gap": "competing implementations: three successive PRs carried the decision before one merged" + }, + { + "id": "ripgrep-long-line-handling", + "source": "thread", + "url": "https://github.com/BurntSushi/ripgrep/issues/129", + "gap": "competing implementations: PRs #245, #356 and #406 each embodied a different answer" + }, + { + "id": "aspnetcore-mvc-in-shared-framework", + "source": "thread", + "url": "https://github.com/dotnet/aspnetcore/issues/3755", + "gap": "private evidence: the deciders cited customer app inspections and other feedback channels that the thread never saw" + } + ] + }, + "crowd-signal": { + "all": { + "k": 48, + "n": 173, + "p": 0.277, + "ci95": [ + 0.216, + 0.348 + ] + }, + "threads": { + "k": 47, + "n": 132, + "p": 0.356, + "ci95": [ + 0.28, + 0.441 + ] + }, + "documents": { + "k": 1, + "n": 41, + "p": 0.024, + "ci95": [ + 0.004, + 0.126 + ] + }, + "small": { + "k": 3, + "n": 11, + "p": 0.273, + "ci95": [ + 0.097, + 0.566 + ] + }, + "entries_total": 49, + "formal_share_of_hits": { + "k": 6, + "n": 48, + "p": 0.125, + "ci95": [ + 0.059, + 0.247 + ] + }, + "examples": [ + { + "id": "just-parallel-mechanism", + "source": "thread", + "url": "https://github.com/casey/just/issues/626", + "gap": "votes as evidence: 83 reactions on the opener and many +1 use cases signalled demand without a structured tally" + }, + { + "id": "just-telemetry-mechanism", + "source": "thread", + "url": "https://github.com/casey/just/discussions/2934", + "gap": "demand signal: whether anyone besides the opener wants the feature was the deciding criterion, but there is no way to gather that interest" + }, + { + "id": "ripgrep-pager-flag", + "source": "thread", + "url": "https://github.com/BurntSushi/ripgrep/issues/86", + "gap": "votes as evidence: hundreds of reactions and repeated me-too comments were the main pressure but were never weighed explicitly" + }, + { + "id": "appflowy-dark-mode-palette", + "source": "thread", + "url": "https://github.com/AppFlowy-IO/AppFlowy/issues/31", + "gap": "votes as evidence: the designer's external straw poll results were never reported back, so the tally behind option 2 is invisible" + } + ] + }, + "delegate": { + "all": { + "k": 45, + "n": 173, + "p": 0.26, + "ci95": [ + 0.2, + 0.33 + ] + }, + "threads": { + "k": 35, + "n": 132, + "p": 0.265, + "ci95": [ + 0.197, + 0.346 + ] + }, + "documents": { + "k": 10, + "n": 41, + "p": 0.244, + "ci95": [ + 0.138, + 0.393 + ] + }, + "small": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "entries_total": 46, + "formal_share_of_hits": { + "k": 17, + "n": 45, + "p": 0.378, + "ci95": [ + 0.251, + 0.524 + ] + }, + "examples": [ + { + "id": "appflowy-dark-mode-palette", + "source": "thread", + "url": "https://github.com/AppFlowy-IO/AppFlowy/issues/31", + "gap": "decider not in thread: the designer deferred to an MVP design the maintainer had made before joining, which is not in the issue" + }, + { + "id": "cargo-feature-doc-field", + "source": "thread", + "url": "https://github.com/rust-lang/rfcs/pull/3485", + "gap": "decision ownership dispute: which teams must approve was itself contested via labels" + }, + { + "id": "deno-config-permissions", + "source": "thread", + "url": "https://github.com/denoland/deno/issues/12763", + "gap": "decider not in thread: the team that blocked #14520 never stated its threat model in the issue despite a user asking" + }, + { + "id": "deno-deny-flags-strict-check", + "source": "thread", + "url": "https://github.com/denoland/deno/pull/19070", + "gap": "decider not in thread: a reviewer approved but waited on the lead maintainer, who appeared only at review checkpoints" + } + ] + }, + "supersede": { + "all": { + "k": 36, + "n": 173, + "p": 0.208, + "ci95": [ + 0.154, + 0.275 + ] + }, + "threads": { + "k": 26, + "n": 132, + "p": 0.197, + "ci95": [ + 0.138, + 0.273 + ] + }, + "documents": { + "k": 10, + "n": 41, + "p": 0.244, + "ci95": [ + 0.138, + 0.393 + ] + }, + "small": { + "k": 2, + "n": 11, + "p": 0.182, + "ci95": [ + 0.051, + 0.477 + ] + }, + "entries_total": 39, + "formal_share_of_hits": { + "k": 13, + "n": 36, + "p": 0.361, + "ci95": [ + 0.225, + 0.524 + ] + }, + "examples": [ + { + "id": "delta-light-dark-detection", + "source": "thread", + "url": "https://github.com/dandavison/delta/issues/447", + "gap": "implementation elsewhere: the deciding work happened in a separate PR thread" + }, + { + "id": "ripgrep-hyperlink-support", + "source": "thread", + "url": "https://github.com/BurntSushi/ripgrep/issues/665", + "gap": "decision moved to another thread: the final design was argued in PR #2483 review, not the issue" + }, + { + "id": "astro-define-vars-scripts", + "source": "thread", + "url": "https://github.com/withastro/roadmap/discussions/1246", + "gap": "decision moved to another thread: further work continues in a new Stage 2 issue" + }, + { + "id": "astro-inline-script-stage-2", + "source": "thread", + "url": "https://github.com/withastro/roadmap/discussions/995", + "gap": "thread handoff: settling moved the conversation to a new issue rather than recording an answer here" + } + ] + }, + "consensus-mechanics": { + "all": { + "k": 32, + "n": 173, + "p": 0.185, + "ci95": [ + 0.134, + 0.249 + ] + }, + "threads": { + "k": 16, + "n": 132, + "p": 0.121, + "ci95": [ + 0.076, + 0.188 + ] + }, + "documents": { + "k": 16, + "n": 41, + "p": 0.39, + "ci95": [ + 0.257, + 0.543 + ] + }, + "small": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "entries_total": 43, + "formal_share_of_hits": { + "k": 21, + "n": 32, + "p": 0.656, + "ci95": [ + 0.483, + 0.796 + ] + }, + "examples": [ + { + "id": "crystal-io-first", + "source": "thread", + "url": "https://github.com/crystal-lang/crystal/issues/5916", + "gap": "unresolved dissent at settlement: the issue closed on merge while a collaborator's objection stood unanswered" + }, + { + "id": "eslint-native-ts-config-flag", + "source": "thread", + "url": "https://github.com/eslint/eslint/issues/19985", + "gap": "decided too fast: closure came within a day, before key stakeholders could weigh in" + }, + { + "id": "excalidraw-new-drawing-button", + "source": "thread", + "url": "https://github.com/excalidraw/excalidraw/issues/1957", + "gap": "contested settled decision: users kept challenging a closed decision for years with no formal reopen path" + }, + { + "id": "go-remove-binary-only-packages", + "source": "thread", + "url": "https://github.com/golang/go/issues/28152", + "gap": "silence as consent: acceptance rested on a week passing with no objections rather than an explicit check of affected users" + } + ] + }, + "nested": { + "all": { + "k": 27, + "n": 173, + "p": 0.156, + "ci95": [ + 0.11, + 0.218 + ] + }, + "threads": { + "k": 14, + "n": 132, + "p": 0.106, + "ci95": [ + 0.064, + 0.17 + ] + }, + "documents": { + "k": 13, + "n": 41, + "p": 0.317, + "ci95": [ + 0.196, + 0.47 + ] + }, + "small": { + "k": 1, + "n": 11, + "p": 0.091, + "ci95": [ + 0.016, + 0.377 + ] + }, + "entries_total": 28, + "formal_share_of_hits": { + "k": 12, + "n": 27, + "p": 0.444, + "ci95": [ + 0.276, + 0.627 + ] + }, + "examples": [ + { + "id": "just-f-string-syntax", + "source": "thread", + "url": "https://github.com/casey/just/issues/11", + "gap": "split scope: settling one part (strings) spun the remainder (backticks) into a new issue" + }, + { + "id": "aspnetcore-mvc-in-shared-framework", + "source": "thread", + "url": "https://github.com/dotnet/aspnetcore/issues/3755", + "gap": "living decision list: the issue served as a running list of removals, each settled in a separate PR, rather than one decision" + }, + { + "id": "deno-rust-type-checker", + "source": "thread", + "url": "https://github.com/denoland/deno/issues/5432", + "gap": "coupled decisions: the umbrella question bundled several staged sub-decisions (no-check default, isolatedModules, emit via swc) each settled in other issues" + }, + { + "id": "excalidraw-export-split-labels", + "source": "thread", + "url": "https://github.com/excalidraw/excalidraw/pull/3613", + "gap": "staged acceptance: several reviewer points were explicitly deferred to follow-up PRs with no tracked record" + } + ] + }, + "revert": { + "all": { + "k": 24, + "n": 173, + "p": 0.139, + "ci95": [ + 0.095, + 0.198 + ] + }, + "threads": { + "k": 21, + "n": 132, + "p": 0.159, + "ci95": [ + 0.106, + 0.231 + ] + }, + "documents": { + "k": 3, + "n": 41, + "p": 0.073, + "ci95": [ + 0.025, + 0.194 + ] + }, + "small": { + "k": 4, + "n": 11, + "p": 0.364, + "ci95": [ + 0.152, + 0.646 + ] + }, + "entries_total": 25, + "formal_share_of_hits": { + "k": 3, + "n": 24, + "p": 0.125, + "ci95": [ + 0.043, + 0.31 + ] + }, + "examples": [ + { + "id": "just-f-string-syntax", + "source": "thread", + "url": "https://github.com/casey/just/issues/11", + "gap": "reversal after merge: the brace choice was flipped in-thread after the implementing PR landed but before release" + }, + { + "id": "ripgrep-hyperlink-support", + "source": "thread", + "url": "https://github.com/BurntSushi/ripgrep/issues/665", + "gap": "reversal after closing: a won't-do decision was reopened years later when a concrete design appeared" + }, + { + "id": "ripgrep-pager-flag", + "source": "thread", + "url": "https://github.com/BurntSushi/ripgrep/issues/86", + "gap": "reversal after decision: settled as no, reopened with a plan and help wanted, then closed again three years later without a build" + }, + { + "id": "zoxide-fish-cd-call", + "source": "thread", + "url": "https://github.com/ajeetdsouza/zoxide/issues/145", + "gap": "reversal after shipping: the first merged fix was changed and reverted in commits with no record in the thread until a contributor noticed" + } + ] + }, + "stale-idle": { + "all": { + "k": 24, + "n": 173, + "p": 0.139, + "ci95": [ + 0.095, + 0.198 + ] + }, + "threads": { + "k": 22, + "n": 132, + "p": 0.167, + "ci95": [ + 0.113, + 0.239 + ] + }, + "documents": { + "k": 2, + "n": 41, + "p": 0.049, + "ci95": [ + 0.013, + 0.161 + ] + }, + "small": { + "k": 4, + "n": 11, + "p": 0.364, + "ci95": [ + 0.152, + 0.646 + ] + }, + "entries_total": 24, + "formal_share_of_hits": { + "k": 4, + "n": 24, + "p": 0.167, + "ci95": [ + 0.067, + 0.359 + ] + }, + "examples": [ + { + "id": "delta-light-dark-detection", + "source": "thread", + "url": "https://github.com/dandavison/delta/issues/447", + "gap": "long dormancy: the question sat for three years between feasibility findings and the fix" + }, + { + "id": "just-f-string-syntax", + "source": "thread", + "url": "https://github.com/casey/just/issues/11", + "gap": "years-long dormancy: a nine-year-old question with stalled community PRs and no record of why it waited" + }, + { + "id": "just-parallel-mechanism", + "source": "thread", + "url": "https://github.com/casey/just/issues/626", + "gap": "long dormancy: five years between framing and decision, with the maintainer's capacity as the real blocker" + }, + { + "id": "just-telemetry-mechanism", + "source": "thread", + "url": "https://github.com/casey/just/discussions/2934", + "gap": "stalled question: a maintainer gave an early lean and then went silent for ten months, with no owner or next step recorded" + } + ] + }, + "implicit": { + "all": { + "k": 22, + "n": 173, + "p": 0.127, + "ci95": [ + 0.085, + 0.185 + ] + }, + "threads": { + "k": 19, + "n": 132, + "p": 0.144, + "ci95": [ + 0.094, + 0.214 + ] + }, + "documents": { + "k": 3, + "n": 41, + "p": 0.073, + "ci95": [ + 0.025, + 0.194 + ] + }, + "small": { + "k": 1, + "n": 11, + "p": 0.091, + "ci95": [ + 0.016, + 0.377 + ] + }, + "entries_total": 24, + "formal_share_of_hits": { + "k": 3, + "n": 22, + "p": 0.136, + "ci95": [ + 0.047, + 0.333 + ] + }, + "examples": [ + { + "id": "fzf-preview-window", + "source": "thread", + "url": "https://github.com/junegunn/fzf/issues/587", + "gap": "prototype-first framing: the maintainer had already built it, so yes/no was settled by demonstration rather than a question" + }, + { + "id": "appflowy-dark-mode-palette", + "source": "thread", + "url": "https://github.com/AppFlowy-IO/AppFlowy/issues/31", + "gap": "staged acceptance: the palette was agreed in December but implementation drifted across contributors and PRs for months, and the closure never links the decision to the code that shipped it" + }, + { + "id": "excalidraw-embed-validator-api", + "source": "thread", + "url": "https://github.com/excalidraw/excalidraw/pull/6691", + "gap": "reviewer implements the answer: the maintainer settled the API by pushing commits to the contributor's branch rather than recording a choice" + }, + { + "id": "ghostty-broadcast-groups-design", + "source": "thread", + "url": "https://github.com/ghostty-org/ghostty/pull/13745", + "gap": "decisions as code suggestions: many design choices were made via suggestion blocks, so the record lives in diffs, not prose" + } + ] + }, + "blocking": { + "all": { + "k": 21, + "n": 173, + "p": 0.121, + "ci95": [ + 0.081, + 0.178 + ] + }, + "threads": { + "k": 14, + "n": 132, + "p": 0.106, + "ci95": [ + 0.064, + 0.17 + ] + }, + "documents": { + "k": 7, + "n": 41, + "p": 0.171, + "ci95": [ + 0.085, + 0.313 + ] + }, + "small": { + "k": 1, + "n": 11, + "p": 0.091, + "ci95": [ + 0.016, + 0.377 + ] + }, + "entries_total": 21, + "formal_share_of_hits": { + "k": 5, + "n": 21, + "p": 0.238, + "ci95": [ + 0.106, + 0.451 + ] + }, + "examples": [ + { + "id": "ripgrep-long-line-handling", + "source": "thread", + "url": "https://github.com/BurntSushi/ripgrep/issues/129", + "gap": "staged acceptance: context-aware display was deferred pending an architecture refactor" + }, + { + "id": "eslint-formatting-rule-deprecation", + "source": "thread", + "url": "https://github.com/eslint/eslint/issues/17522", + "gap": "external dependency: the decision waited on a separate community project being ready" + }, + { + "id": "excalidraw-new-drawing-button", + "source": "thread", + "url": "https://github.com/excalidraw/excalidraw/issues/1957", + "gap": "coupled decisions: the answer depended on a separate multi-scene issue that was never resolved here" + }, + { + "id": "ghostty-broadcast-groups-design", + "source": "thread", + "url": "https://github.com/ghostty-org/ghostty/pull/13745", + "gap": "settled but not shipped: approved yet held for a release string freeze, so settled and merged diverge" + } + ] + }, + "reframe": { + "all": { + "k": 20, + "n": 173, + "p": 0.116, + "ci95": [ + 0.076, + 0.172 + ] + }, + "threads": { + "k": 14, + "n": 132, + "p": 0.106, + "ci95": [ + 0.064, + 0.17 + ] + }, + "documents": { + "k": 6, + "n": 41, + "p": 0.146, + "ci95": [ + 0.069, + 0.284 + ] + }, + "small": { + "k": 1, + "n": 11, + "p": 0.091, + "ci95": [ + 0.016, + 0.377 + ] + }, + "entries_total": 21, + "formal_share_of_hits": { + "k": 7, + "n": 20, + "p": 0.35, + "ci95": [ + 0.181, + 0.567 + ] + }, + "examples": [ + { + "id": "just-telemetry-mechanism", + "source": "thread", + "url": "https://github.com/casey/just/discussions/2934", + "gap": "reframing: the question shifted from 'add telemetry' to 'add a generic event hook', which needs the question itself to be renamed rather than answered" + }, + { + "id": "excalidraw-handwritten-font", + "source": "thread", + "url": "https://github.com/excalidraw/excalidraw/issues/2945", + "gap": "reframed question: the final answer (improve the incumbent) was not one of the options in the original framing" + }, + { + "id": "go-try-builtin", + "source": "thread", + "url": "https://github.com/golang/go/issues/32437", + "gap": "problem before solution: the team conceded the problem statement was never agreed, so a question about whether to solve anything was hidden inside a question about one design" + }, + { + "id": "ha-addons-to-apps", + "source": "thread", + "url": "https://github.com/home-assistant/architecture/discussions/1287", + "gap": "reframing: many replies argued the real problem was a different menu, which the design cannot split off" + } + ] + }, + "consequences": { + "all": { + "k": 20, + "n": 173, + "p": 0.116, + "ci95": [ + 0.076, + 0.172 + ] + }, + "threads": { + "k": 5, + "n": 132, + "p": 0.038, + "ci95": [ + 0.016, + 0.086 + ] + }, + "documents": { + "k": 15, + "n": 41, + "p": 0.366, + "ci95": [ + 0.236, + 0.519 + ] + }, + "small": { + "k": 1, + "n": 11, + "p": 0.091, + "ci95": [ + 0.016, + 0.377 + ] + }, + "entries_total": 24, + "formal_share_of_hits": { + "k": 4, + "n": 20, + "p": 0.2, + "ci95": [ + 0.081, + 0.416 + ] + }, + "examples": [ + { + "id": "fzf-default-opts-file", + "source": "thread", + "url": "https://github.com/junegunn/fzf/discussions/2528", + "gap": "follow-up defect: a missed integration (key bindings) surfaced after the decision and needed its own PR" + }, + { + "id": "eslint-adopt-codemod", + "source": "thread", + "url": "https://github.com/eslint/eslint/issues/20442", + "gap": "staged acceptance: deprecation of the old tools was agreed but not yet carried out" + }, + { + "id": "ha-zwave-driver-choice", + "source": "thread", + "url": "https://github.com/home-assistant/architecture/issues/483", + "gap": "spun-off follow-up: migration and old-integration retirement were deferred to separate issues" + }, + { + "id": "home-assistant-lawn-mower-entity-model", + "source": "thread", + "url": "https://github.com/home-assistant/architecture/discussions/894", + "gap": "cross-repo build: one decision fanned out to core, frontend, brands and two docs PRs" + } + ] + }, + "conditional": { + "all": { + "k": 18, + "n": 173, + "p": 0.104, + "ci95": [ + 0.067, + 0.158 + ] + }, + "threads": { + "k": 9, + "n": 132, + "p": 0.068, + "ci95": [ + 0.036, + 0.125 + ] + }, + "documents": { + "k": 9, + "n": 41, + "p": 0.22, + "ci95": [ + 0.12, + 0.367 + ] + }, + "small": { + "k": 1, + "n": 11, + "p": 0.091, + "ci95": [ + 0.016, + 0.377 + ] + }, + "entries_total": 19, + "formal_share_of_hits": { + "k": 6, + "n": 18, + "p": 0.333, + "ci95": [ + 0.163, + 0.563 + ] + }, + "examples": [ + { + "id": "bat-builtin-pager-approach", + "source": "thread", + "url": "https://github.com/sharkdp/bat/issues/1053", + "gap": "reversal after shipping: post-ship evidence (unmaintained upstream, large-file failures) reopens the library choice without a formal revisit" + }, + { + "id": "bat-debian-batcat-name", + "source": "thread", + "url": "https://github.com/sharkdp/bat/issues/2455", + "gap": "stale facts: no prompt to re-verify a 'True' check when upstream changes later" + }, + { + "id": "eslint-native-ts-config-flag", + "source": "thread", + "url": "https://github.com/eslint/eslint/issues/19985", + "gap": "stale rationale: the stated condition for revisiting depended on external Node behaviour that later changed, with nothing prompting the decider to re-check" + }, + { + "id": "ghostty-official-snap", + "source": "thread", + "url": "https://github.com/ghostty-org/ghostty/pull/3931", + "gap": "conditional acceptance: the yes depended on the contributor committing to future maintenance, a people commitment the record cannot capture" + } + ] + }, + "external-authority": { + "all": { + "k": 17, + "n": 173, + "p": 0.098, + "ci95": [ + 0.062, + 0.152 + ] + }, + "threads": { + "k": 8, + "n": 132, + "p": 0.061, + "ci95": [ + 0.031, + 0.115 + ] + }, + "documents": { + "k": 9, + "n": 41, + "p": 0.22, + "ci95": [ + 0.12, + 0.367 + ] + }, + "small": { + "k": 2, + "n": 11, + "p": 0.182, + "ci95": [ + 0.051, + 0.477 + ] + }, + "entries_total": 17, + "formal_share_of_hits": { + "k": 4, + "n": 17, + "p": 0.235, + "ci95": [ + 0.096, + 0.473 + ] + }, + "examples": [ + { + "id": "ripgrep-hyperlink-support", + "source": "thread", + "url": "https://github.com/BurntSushi/ripgrep/issues/665", + "gap": "external dependency evidence: Windows Terminal and tmux fixes upstream shaped what was viable" + }, + { + "id": "ripgrep-pager-flag", + "source": "thread", + "url": "https://github.com/BurntSushi/ripgrep/issues/86", + "gap": "external alternative as answer: the outcome leans on a third-party tool (delta) endorsed in the README rather than a built feature" + }, + { + "id": "bat-debian-batcat-name", + "source": "thread", + "url": "https://github.com/sharkdp/bat/issues/2455", + "gap": "decider not in thread: the real deciders are Debian and Bacula maintainers, and bat's own maintainers spoke only once" + }, + { + "id": "deno-jupyter-display-hook", + "source": "thread", + "url": "https://github.com/denoland/deno/pull/20337", + "gap": "external expert input: a Jupyter maintainer from outside the project steered the design, a role the design does not distinguish" + } + ] + }, + "approve": { + "all": { + "k": 14, + "n": 173, + "p": 0.081, + "ci95": [ + 0.049, + 0.131 + ] + }, + "threads": { + "k": 12, + "n": 132, + "p": 0.091, + "ci95": [ + 0.053, + 0.152 + ] + }, + "documents": { + "k": 2, + "n": 41, + "p": 0.049, + "ci95": [ + 0.013, + 0.161 + ] + }, + "small": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "entries_total": 14, + "formal_share_of_hits": { + "k": 7, + "n": 14, + "p": 0.5, + "ci95": [ + 0.268, + 0.732 + ] + }, + "examples": [ + { + "id": "astro-define-vars-scripts", + "source": "thread", + "url": "https://github.com/withastro/roadmap/discussions/1246", + "gap": "per-member approvals: each TSC member's sign-off is a separate reply that the record should tally" + }, + { + "id": "astro-inline-script-stage-2", + "source": "thread", + "url": "https://github.com/withastro/roadmap/discussions/995", + "gap": "votes as evidence: steering committee members approve with one-line replies counted as a gate" + }, + { + "id": "cargo-feature-doc-field", + "source": "thread", + "url": "https://github.com/rust-lang/rfcs/pull/3485", + "gap": "formal sign-off mechanics: rfcbot checkboxes and concerns act as the settle step, which Chopin's single settler does not model" + }, + { + "id": "ghostty-broadcast-groups-design", + "source": "thread", + "url": "https://github.com/ghostty-org/ghostty/pull/13745", + "gap": "multiple approvers per platform: GTK and macOS maintainers each signed off on their area" + } + ] + }, + "rejected-content": { + "all": { + "k": 13, + "n": 173, + "p": 0.075, + "ci95": [ + 0.044, + 0.124 + ] + }, + "threads": { + "k": 3, + "n": 132, + "p": 0.023, + "ci95": [ + 0.008, + 0.065 + ] + }, + "documents": { + "k": 10, + "n": 41, + "p": 0.244, + "ci95": [ + 0.138, + 0.393 + ] + }, + "small": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "entries_total": 13, + "formal_share_of_hits": { + "k": 7, + "n": 13, + "p": 0.538, + "ci95": [ + 0.291, + 0.768 + ] + }, + "examples": [ + { + "id": "deno-config-permissions", + "source": "thread", + "url": "https://github.com/denoland/deno/issues/12763", + "gap": "reversal after shipping: two implementation PRs were opened and abandoned before the final design, with the reason recorded only on a closed PR" + }, + { + "id": "ha-entity-name-editing", + "source": "thread", + "url": "https://github.com/home-assistant/architecture/discussions/1185", + "gap": "silent option removal: an option added in response to feedback was later dropped by editing the proposal, with no recorded reasoning" + }, + { + "id": "ha-template-binary-sensor-delays-ui", + "source": "thread", + "url": "https://github.com/home-assistant/architecture/discussions/1340", + "gap": "reopening a decision scattered across PRs: the prior refusal lived in five closed PRs rather than one record" + }, + { + "id": "eng-govuk-infrastructure-adr019-restructure-cicd", + "source": "document", + "url": "https://github.com/alphagov/govuk-infrastructure/blob/main/docs/architecture/decisions/0019-restructure-cicd.md", + "gap": "rejected options content: A late option with no recorded reason for non-adoption: an option whose fate has no rationale." + } + ] + }, + "convention": { + "all": { + "k": 12, + "n": 173, + "p": 0.069, + "ci95": [ + 0.04, + 0.117 + ] + }, + "threads": { + "k": 3, + "n": 132, + "p": 0.023, + "ci95": [ + 0.008, + 0.065 + ] + }, + "documents": { + "k": 9, + "n": 41, + "p": 0.22, + "ci95": [ + 0.12, + 0.367 + ] + }, + "small": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "entries_total": 13, + "formal_share_of_hits": { + "k": 5, + "n": 12, + "p": 0.417, + "ci95": [ + 0.193, + 0.68 + ] + }, + "examples": [ + { + "id": "excalidraw-zen-mode-entry-exit", + "source": "thread", + "url": "https://github.com/excalidraw/excalidraw/pull/1450", + "gap": "decider not in thread: the team had no written design philosophy, so the author built a dialog that reviewers then rejected" + }, + { + "id": "prettier-final-newline", + "source": "thread", + "url": "https://github.com/prettier/prettier/issues/6360", + "gap": "standing policy as decider: the outcome follows a project-wide option philosophy document rather than this thread's arguments, and the record needs to cite it" + }, + { + "id": "prettier-nested-ternary-format", + "source": "thread", + "url": "https://github.com/prettier/prettier/issues/9561", + "gap": "principle constraint: a standing no-options policy ruled out a popular answer, which needs to sit beside the question as a fixed constraint" + }, + { + "id": "eng-backstage-adr013-014-http-fetch-library", + "source": "document", + "url": "https://github.com/backstage/backstage/blob/master/docs/architecture-decisions/adr014-use-fetch.md", + "gap": "convention scope: One decision with three scopes (Node.js code, frontend, isomorphic packages), only one of which changed. The choice table holds a single answer, not per-context answers where one cel" + } + ] + }, + "vote": { + "all": { + "k": 10, + "n": 173, + "p": 0.058, + "ci95": [ + 0.032, + 0.103 + ] + }, + "threads": { + "k": 3, + "n": 132, + "p": 0.023, + "ci95": [ + 0.008, + 0.065 + ] + }, + "documents": { + "k": 7, + "n": 41, + "p": 0.171, + "ci95": [ + 0.085, + 0.313 + ] + }, + "small": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "entries_total": 11, + "formal_share_of_hits": { + "k": 7, + "n": 10, + "p": 0.7, + "ci95": [ + 0.397, + 0.892 + ] + }, + "examples": [ + { + "id": "node-primordials-error-path", + "source": "thread", + "url": "https://github.com/nodejs/TSC/issues/1104", + "gap": "votes or reaction counts as evidence: charter seconding was counted from thumbs-up reactions, and the decision itself was a ranked Condorcet ballot" + }, + { + "id": "node-yarn-vs-corepack", + "source": "thread", + "url": "https://github.com/nodejs/TSC/issues/1012", + "gap": "votes as evidence: the decision was a formal ranked ballot by a closed electorate with private ballots and turnout tracking, not thumbs from whoever passes by" + }, + { + "id": "zed-git-panel-file-diff", + "source": "thread", + "url": "https://github.com/zed-industries/zed/discussions/33773", + "gap": "votes as evidence: the default was settled by a quick poll of reply votes from the implementer" + }, + { + "id": "eng-modernisation-platform-adr004-019-core-languages", + "source": "document", + "url": "https://github.com/ministryofjustice/modernisation-platform/blob/main/architecture-decision-record/0019-use-bash-go-as-core-languages.md", + "gap": "formal vote: A poll feeding a decision, tied 3-3, followed by a rule to let each member pick. The design has Ask and Takes (Agrees/Objects/Unsure) but no poll tally, no quorum, and no 'decide not to d" + } + ] + }, + "deferred": { + "all": { + "k": 9, + "n": 173, + "p": 0.052, + "ci95": [ + 0.028, + 0.096 + ] + }, + "threads": { + "k": 4, + "n": 132, + "p": 0.03, + "ci95": [ + 0.012, + 0.075 + ] + }, + "documents": { + "k": 5, + "n": 41, + "p": 0.122, + "ci95": [ + 0.053, + 0.255 + ] + }, + "small": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "entries_total": 9, + "formal_share_of_hits": { + "k": 5, + "n": 9, + "p": 0.556, + "ci95": [ + 0.267, + 0.811 + ] + }, + "examples": [ + { + "id": "lazygit-worktree-naming-location", + "source": "thread", + "url": "https://github.com/jesseduffield/lazygit/discussions/2803", + "gap": "deferred question never reopened: later requests for name derivation pile up in the same thread without reviving the deferred item" + }, + { + "id": "mastodon-local-only-posts", + "source": "thread", + "url": "https://github.com/mastodon/mastodon/issues/861", + "gap": "limbo state: neither accepted nor rejected, only signalled by label changes" + }, + { + "id": "outline-document-tags", + "source": "thread", + "url": "https://github.com/outline/outline/discussions/2256", + "gap": "staged acceptance: a 'not at this time' deferral has no trigger or review date for reopening" + }, + { + "id": "vite-lib-externalize-lookup", + "source": "thread", + "url": "https://github.com/vitejs/vite/pull/10632", + "gap": "deferred to a larger initiative: maintainers parked it for a future library-mode review that is not linked or tracked" + } + ] + }, + "partition": { + "all": { + "k": 8, + "n": 173, + "p": 0.046, + "ci95": [ + 0.024, + 0.089 + ] + }, + "threads": { + "k": 8, + "n": 132, + "p": 0.061, + "ci95": [ + 0.031, + 0.115 + ] + }, + "documents": { + "k": 0, + "n": 41, + "p": 0.0, + "ci95": [ + 0, + 0.086 + ] + }, + "small": { + "k": 2, + "n": 11, + "p": 0.182, + "ci95": [ + 0.051, + 0.477 + ] + }, + "entries_total": 8, + "formal_share_of_hits": { + "k": 3, + "n": 8, + "p": 0.375, + "ci95": [ + 0.137, + 0.694 + ] + }, + "examples": [ + { + "id": "delta-light-dark-detection", + "source": "thread", + "url": "https://github.com/dandavison/delta/issues/447", + "gap": "residual limits after settling: tmux and lazygit cases stayed unsolved after closing" + }, + { + "id": "fzf-preview-window", + "source": "thread", + "url": "https://github.com/junegunn/fzf/issues/587", + "gap": "scope guarding after close: follow-on option requests were refused in the closed thread, a sub-decision the record would miss" + }, + { + "id": "astro-scripts-only-when-rendered", + "source": "thread", + "url": "https://github.com/withastro/roadmap/discussions/664", + "gap": "coupled decisions: scripts and styles were treated as one question, then split when the maintainer judged styles differently" + }, + { + "id": "eslint-formatting-rule-deprecation", + "source": "thread", + "url": "https://github.com/eslint/eslint/issues/17522", + "gap": "per-item settlement: the answer was a list of about seventy rules each with its own keep-or-drop call" + } + ] + }, + "provisional": { + "all": { + "k": 7, + "n": 173, + "p": 0.04, + "ci95": [ + 0.02, + 0.081 + ] + }, + "threads": { + "k": 1, + "n": 132, + "p": 0.008, + "ci95": [ + 0.001, + 0.042 + ] + }, + "documents": { + "k": 6, + "n": 41, + "p": 0.146, + "ci95": [ + 0.069, + 0.284 + ] + }, + "small": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "entries_total": 7, + "formal_share_of_hits": { + "k": 3, + "n": 7, + "p": 0.429, + "ci95": [ + 0.158, + 0.75 + ] + }, + "examples": [ + { + "id": "prettier-nested-ternary-format", + "source": "thread", + "url": "https://github.com/prettier/prettier/issues/9561", + "gap": "experiments as evidence: the outcome so far is an opt-in experimental flag announced on a blog, not a settled answer" + }, + { + "id": "eng-backstage-adr013-014-http-fetch-library", + "source": "document", + "url": "https://github.com/backstage/backstage/blob/master/docs/architecture-decisions/adr014-use-fetch.md", + "gap": "provisional decision: Reversal trigger is an environmental change (minimum platform version raised), not a new argument. The record has 'What we knew at the time' but nothing like 'revisit when X chan" + }, + { + "id": "eng-cloud-platform-adr012-021-036-048-multi-cluster", + "source": "document", + "url": "https://github.com/ministryofjustice/cloud-platform/blob/main/architecture-decision-record/036-multi-cluster.md", + "gap": "provisional decision: Evidence is a dated snapshot ('at time of writing' namespace counts, comparison against Kubernetes large-cluster limits). 'What we knew' could hold it, but nothing flags it for r" + }, + { + "id": "eng-modernisation-platform-adr004-019-core-languages", + "source": "document", + "url": "https://github.com/ministryofjustice/modernisation-platform/blob/main/architecture-decision-record/0019-use-bash-go-as-core-languages.md", + "gap": "provisional decision: Both ADRs say they can be revisited, but neither names a review date or trigger. The design has no 'revisit when' condition on a settled question." + } + ] + }, + "visual-comparison": { + "all": { + "k": 5, + "n": 173, + "p": 0.029, + "ci95": [ + 0.012, + 0.066 + ] + }, + "threads": { + "k": 4, + "n": 132, + "p": 0.03, + "ci95": [ + 0.012, + 0.075 + ] + }, + "documents": { + "k": 1, + "n": 41, + "p": 0.024, + "ci95": [ + 0.004, + 0.126 + ] + }, + "small": { + "k": 1, + "n": 11, + "p": 0.091, + "ci95": [ + 0.016, + 0.377 + ] + }, + "entries_total": 5, + "formal_share_of_hits": { + "k": 1, + "n": 5, + "p": 0.2, + "ci95": [ + 0.036, + 0.624 + ] + }, + "examples": [ + { + "id": "fzf-preview-window", + "source": "thread", + "url": "https://github.com/junegunn/fzf/issues/587", + "gap": "screenshots as evidence: layout choices were argued with terminal screenshots, not a structured comparison" + }, + { + "id": "excalidraw-handwritten-font", + "source": "thread", + "url": "https://github.com/excalidraw/excalidraw/issues/2945", + "gap": "visual evidence: options were judged from rendered samples and a live preview site rather than text" + }, + { + "id": "vscode-completion-item-label-shape", + "source": "thread", + "url": "https://github.com/microsoft/vscode/issues/39441", + "gap": "visual alternatives as evidence: competing renderings were compared through screenshots" + }, + { + "id": "zed-commit-log-graph", + "source": "thread", + "url": "https://github.com/zed-industries/zed/discussions/26511", + "gap": "visual references as evidence: options arrived as screenshots of other tools, with no way to compare them side by side" + } + ] + }, + "other:hidden-history": { + "all": { + "k": 3, + "n": 173, + "p": 0.017, + "ci95": [ + 0.006, + 0.05 + ] + }, + "threads": { + "k": 3, + "n": 132, + "p": 0.023, + "ci95": [ + 0.008, + 0.065 + ] + }, + "documents": { + "k": 0, + "n": 41, + "p": 0.0, + "ci95": [ + 0, + 0.086 + ] + }, + "small": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "entries_total": 3, + "formal_share_of_hits": { + "k": 0, + "n": 3, + "p": 0.0, + "ci95": [ + 0, + 0.562 + ] + }, + "examples": [ + { + "id": "eslint-formatting-rule-deprecation", + "source": "thread", + "url": "https://github.com/eslint/eslint/issues/17522", + "gap": "thread moderation: locking and unlocking to steer scope has no equivalent" + }, + { + "id": "typescript-node-esm-mode-shape", + "source": "thread", + "url": "https://github.com/microsoft/TypeScript/pull/44501", + "gap": "duplicates merged: repeated objections were hidden as duplicates rather than linked to the settled record" + }, + { + "id": "zed-git-panel-file-diff", + "source": "thread", + "url": "https://github.com/zed-industries/zed/discussions/33773", + "gap": "existing-capability check: a hidden existing action partly answered the request, a fact that needed checking before building" + } + ] + }, + "moot": { + "all": { + "k": 3, + "n": 173, + "p": 0.017, + "ci95": [ + 0.006, + 0.05 + ] + }, + "threads": { + "k": 1, + "n": 132, + "p": 0.008, + "ci95": [ + 0.001, + 0.042 + ] + }, + "documents": { + "k": 2, + "n": 41, + "p": 0.049, + "ci95": [ + 0.013, + 0.161 + ] + }, + "small": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "entries_total": 3, + "formal_share_of_hits": { + "k": 1, + "n": 3, + "p": 0.333, + "ci95": [ + 0.061, + 0.792 + ] + }, + "examples": [ + { + "id": "mastodon-remote-local-timeline", + "source": "thread", + "url": "https://github.com/mastodon/mastodon/issues/1053", + "gap": "partial solution as closure: a different feature that met part of the need was used to close the request" + }, + { + "id": "eng-govuk-infrastructure-adr007-application-logging", + "source": "document", + "url": "https://github.com/alphagov/govuk-infrastructure/blob/main/docs/architecture/decisions/0007-use-fluentbit-elasticsearch-and-kibana-for-application-logs.md", + "gap": "moot question: Deprecated with no successor: the decision was withdrawn rather than replaced, and the repo does not say what is now true. Settled and Reopen exist, but there is no 'retired, no replace" + }, + { + "id": "tc39-2023-11-operator-overloading-withdrawn", + "source": "document", + "url": "https://github.com/tc39/notes/blob/main/meetings/2023-11/november-28.md#withdrawing-operator-overloading", + "gap": "moot question: A negative decision: the outcome removes a feature and closes a line of work. Settle writes an answer sentence into the plan, and nothing records withdrawn, dropped or not doing it." + } + ] + }, + "other:deadline": { + "all": { + "k": 3, + "n": 173, + "p": 0.017, + "ci95": [ + 0.006, + 0.05 + ] + }, + "threads": { + "k": 0, + "n": 132, + "p": 0.0, + "ci95": [ + 0, + 0.028 + ] + }, + "documents": { + "k": 3, + "n": 41, + "p": 0.073, + "ci95": [ + 0.025, + 0.194 + ] + }, + "small": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "entries_total": 3, + "formal_share_of_hits": { + "k": 3, + "n": 3, + "p": 1.0, + "ci95": [ + 0.438, + 1 + ] + }, + "examples": [ + { + "id": "tc39-2020-04-weakrefs-finalizationregistry-cleanup", + "source": "document", + "url": "https://github.com/tc39/notes/blob/main/meetings/2020-03/april-1.md#weakrefs-finalizationregistry-api-change", + "gap": "deadline: Implementation and a Test262 PR were already waiting, so the decision carried shipping urgency that the design cannot express." + }, + { + "id": "tc39-2021-08-realm-constructor-shadowrealm-name", + "source": "document", + "url": "https://github.com/tc39/notes/blob/main/meetings/2021-08/aug-31.md#realms-renaming-bikeshedding-thread", + "gap": "deadline: The thread ended because patience ran out and a delegate called time; there is no discussion-has-run-its-course affordance or deadline on a question." + }, + { + "id": "tc39-2025-09-iterator-windows-undersized-parameter", + "source": "document", + "url": "https://github.com/tc39/notes/blob/main/meetings/2025-05/may-29.md#iterator-chunking-for-stage-27", + "gap": "deadline: Feedback that arrived days before a meeting changed the design and the champion said it should have come earlier; the design has no early-input nudge or feedback deadline." + } + ] + }, + "ratify": { + "all": { + "k": 2, + "n": 173, + "p": 0.012, + "ci95": [ + 0.003, + 0.041 + ] + }, + "threads": { + "k": 2, + "n": 132, + "p": 0.015, + "ci95": [ + 0.004, + 0.054 + ] + }, + "documents": { + "k": 0, + "n": 41, + "p": 0.0, + "ci95": [ + 0, + 0.086 + ] + }, + "small": { + "k": 1, + "n": 11, + "p": 0.091, + "ci95": [ + 0.016, + 0.377 + ] + }, + "entries_total": 2, + "formal_share_of_hits": { + "k": 0, + "n": 2, + "p": 0.0, + "ci95": [ + 0, + 0.658 + ] + }, + "examples": [ + { + "id": "fzf-default-opts-file", + "source": "thread", + "url": "https://github.com/junegunn/fzf/discussions/2528", + "gap": "consultation skipped: the maintainer asked for opinions and shipped the PR before anyone could reply" + }, + { + "id": "ha-floors-vs-nested-areas", + "source": "thread", + "url": "https://github.com/home-assistant/architecture/discussions/1021", + "gap": "decided before discussed: the proposal named its decision and PRs were open before feedback arrived" + } + ] + }, + "rank": { + "all": { + "k": 2, + "n": 173, + "p": 0.012, + "ci95": [ + 0.003, + 0.041 + ] + }, + "threads": { + "k": 0, + "n": 132, + "p": 0.0, + "ci95": [ + 0, + 0.028 + ] + }, + "documents": { + "k": 2, + "n": 41, + "p": 0.049, + "ci95": [ + 0.013, + 0.161 + ] + }, + "small": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "entries_total": 2, + "formal_share_of_hits": { + "k": 2, + "n": 2, + "p": 1.0, + "ci95": [ + 0.342, + 1 + ] + }, + "examples": [ + { + "id": "python-2018-12-governance-model-vote-pep-8016", + "source": "document", + "url": "https://peps.python.org/pep-8001/", + "gap": "ranked choice: A formal ranked vote among full competing documents (Condorcet over seven candidates plus a 'none yet' sentinel), run by an outside trusted party. The nearest primitive is a stance pill" + }, + { + "id": "rust-lang-team-2022-03-lang-roadmap-themes", + "source": "document", + "url": "https://github.com/rust-lang/lang-team/blob/c45345e32e81e273aab6c805b0d3cd7b1a666045/design-meeting-minutes/2022-03-09-lang-roadmap.md", + "gap": "ranked choice: Ordering of items is a decision type: neither a single choice nor a tune dial. There is no ranked-list shape." + } + ] + } + }, + "tag_reliability": { + "n": 60, + "agree": 42, + "po": 0.7, + "pe": 0.068, + "kappa": 0.678, + "confusions": { + "provisional->conditional": 1, + "convention->partition": 1, + "consensus-mechanics->delegate": 1, + "revert->evidence-not-check": 1, + "consequences->rejected-content": 1, + "rejected-content->other": 1, + "coupled->other": 1, + "rank->vote": 1, + "other->reframe": 1, + "blocking->delegate": 1 + } + }, + "gap_groups": { + "G7 follow-ups: nested|staged|consequences": { + "all": { + "k": 87, + "n": 173, + "p": 0.503, + "ci95": [ + 0.429, + 0.577 + ] + }, + "threads": { + "k": 59, + "n": 132, + "p": 0.447, + "ci95": [ + 0.365, + 0.532 + ] + }, + "documents": { + "k": 28, + "n": 41, + "p": 0.683, + "ci95": [ + 0.53, + 0.804 + ] + }, + "small": { + "k": 5, + "n": 11, + "p": 0.455, + "ci95": [ + 0.213, + 0.72 + ] + }, + "formal_share_of_hits": { + "k": 33, + "n": 87, + "p": 0.379, + "ci95": [ + 0.285, + 0.484 + ] + }, + "examples": [ + "bat-builtin-pager-approach", + "delta-light-dark-detection", + "fzf-default-opts-file", + "just-f-string-syntax" + ] + }, + "G9' composite answer (>=2 chosen or partly/merged)": { + "all": { + "k": 106, + "n": 173, + "p": 0.613, + "ci95": [ + 0.538, + 0.682 + ] + }, + "threads": { + "k": 83, + "n": 132, + "p": 0.629, + "ci95": [ + 0.544, + 0.706 + ] + }, + "documents": { + "k": 23, + "n": 41, + "p": 0.561, + "ci95": [ + 0.41, + 0.701 + ] + }, + "small": { + "k": 8, + "n": 11, + "p": 0.727, + "ci95": [ + 0.434, + 0.903 + ] + }, + "formal_share_of_hits": { + "k": 26, + "n": 106, + "p": 0.245, + "ci95": [ + 0.173, + 0.335 + ] + }, + "examples": [ + "bat-builtin-pager-approach", + "delta-light-dark-detection", + "fzf-preview-window", + "just-f-string-syntax" + ] + }, + "G2 parked: deferred|stale-idle or undecided+deferred": { + "all": { + "k": 42, + "n": 173, + "p": 0.243, + "ci95": [ + 0.185, + 0.312 + ] + }, + "threads": { + "k": 30, + "n": 132, + "p": 0.227, + "ci95": [ + 0.164, + 0.306 + ] + }, + "documents": { + "k": 12, + "n": 41, + "p": 0.293, + "ci95": [ + 0.176, + 0.445 + ] + }, + "small": { + "k": 4, + "n": 11, + "p": 0.364, + "ci95": [ + 0.152, + 0.646 + ] + }, + "formal_share_of_hits": { + "k": 11, + "n": 42, + "p": 0.262, + "ci95": [ + 0.153, + 0.411 + ] + }, + "examples": [ + "delta-light-dark-detection", + "just-f-string-syntax", + "just-parallel-mechanism", + "just-telemetry-mechanism" + ] + }, + "G6 evidence beyond check": { + "all": { + "k": 49, + "n": 173, + "p": 0.283, + "ci95": [ + 0.221, + 0.355 + ] + }, + "threads": { + "k": 40, + "n": 132, + "p": 0.303, + "ci95": [ + 0.231, + 0.386 + ] + }, + "documents": { + "k": 9, + "n": 41, + "p": 0.22, + "ci95": [ + 0.12, + 0.367 + ] + }, + "small": { + "k": 3, + "n": 11, + "p": 0.273, + "ci95": [ + 0.097, + 0.566 + ] + }, + "formal_share_of_hits": { + "k": 10, + "n": 49, + "p": 0.204, + "ci95": [ + 0.115, + 0.336 + ] + }, + "examples": [ + "just-parallel-mechanism", + "ripgrep-hyperlink-support", + "ripgrep-long-line-handling", + "aspnetcore-mvc-in-shared-framework" + ] + }, + "G3 decided elsewhere: venue-split|implicit|ratify": { + "all": { + "k": 91, + "n": 173, + "p": 0.526, + "ci95": [ + 0.452, + 0.599 + ] + }, + "threads": { + "k": 73, + "n": 132, + "p": 0.553, + "ci95": [ + 0.468, + 0.635 + ] + }, + "documents": { + "k": 18, + "n": 41, + "p": 0.439, + "ci95": [ + 0.299, + 0.59 + ] + }, + "small": { + "k": 3, + "n": 11, + "p": 0.273, + "ci95": [ + 0.097, + 0.566 + ] + }, + "formal_share_of_hits": { + "k": 26, + "n": 91, + "p": 0.286, + "ci95": [ + 0.203, + 0.386 + ] + }, + "examples": [ + "bat-builtin-pager-approach", + "fzf-default-opts-file", + "fzf-preview-window", + "appflowy-dark-mode-palette" + ] + }, + "P8 external authority": { + "all": { + "k": 17, + "n": 173, + "p": 0.098, + "ci95": [ + 0.062, + 0.152 + ] + }, + "threads": { + "k": 8, + "n": 132, + "p": 0.061, + "ci95": [ + 0.031, + 0.115 + ] + }, + "documents": { + "k": 9, + "n": 41, + "p": 0.22, + "ci95": [ + 0.12, + 0.367 + ] + }, + "small": { + "k": 2, + "n": 11, + "p": 0.182, + "ci95": [ + 0.051, + 0.477 + ] + }, + "formal_share_of_hits": { + "k": 4, + "n": 17, + "p": 0.235, + "ci95": [ + 0.096, + 0.473 + ] + }, + "examples": [ + "ripgrep-hyperlink-support", + "ripgrep-pager-flag", + "bat-debian-batcat-name", + "deno-jupyter-display-hook" + ] + }, + "G5 provisional|conditional": { + "all": { + "k": 24, + "n": 173, + "p": 0.139, + "ci95": [ + 0.095, + 0.198 + ] + }, + "threads": { + "k": 10, + "n": 132, + "p": 0.076, + "ci95": [ + 0.042, + 0.134 + ] + }, + "documents": { + "k": 14, + "n": 41, + "p": 0.341, + "ci95": [ + 0.216, + 0.495 + ] + }, + "small": { + "k": 1, + "n": 11, + "p": 0.091, + "ci95": [ + 0.016, + 0.377 + ] + }, + "formal_share_of_hits": { + "k": 8, + "n": 24, + "p": 0.333, + "ci95": [ + 0.18, + 0.533 + ] + }, + "examples": [ + "bat-builtin-pager-approach", + "bat-debian-batcat-name", + "eslint-native-ts-config-flag", + "ghostty-official-snap" + ] + }, + "G10 supersede|revert": { + "all": { + "k": 55, + "n": 173, + "p": 0.318, + "ci95": [ + 0.253, + 0.391 + ] + }, + "threads": { + "k": 43, + "n": 132, + "p": 0.326, + "ci95": [ + 0.252, + 0.41 + ] + }, + "documents": { + "k": 12, + "n": 41, + "p": 0.293, + "ci95": [ + 0.176, + 0.445 + ] + }, + "small": { + "k": 5, + "n": 11, + "p": 0.455, + "ci95": [ + 0.213, + 0.72 + ] + }, + "formal_share_of_hits": { + "k": 16, + "n": 55, + "p": 0.291, + "ci95": [ + 0.188, + 0.421 + ] + }, + "examples": [ + "delta-light-dark-detection", + "just-f-string-syntax", + "ripgrep-hyperlink-support", + "ripgrep-pager-flag" + ] + }, + "G4 coupled|blocking": { + "all": { + "k": 66, + "n": 173, + "p": 0.382, + "ci95": [ + 0.312, + 0.456 + ] + }, + "threads": { + "k": 52, + "n": 132, + "p": 0.394, + "ci95": [ + 0.315, + 0.479 + ] + }, + "documents": { + "k": 14, + "n": 41, + "p": 0.341, + "ci95": [ + 0.216, + 0.495 + ] + }, + "small": { + "k": 2, + "n": 11, + "p": 0.182, + "ci95": [ + 0.051, + 0.477 + ] + }, + "formal_share_of_hits": { + "k": 20, + "n": 66, + "p": 0.303, + "ci95": [ + 0.206, + 0.422 + ] + }, + "examples": [ + "just-f-string-syntax", + "ripgrep-long-line-handling", + "appflowy-dark-mode-palette", + "astro-define-vars-scripts" + ] + }, + "P10 crowd signal (votes as evidence)": { + "all": { + "k": 48, + "n": 173, + "p": 0.277, + "ci95": [ + 0.216, + 0.348 + ] + }, + "threads": { + "k": 47, + "n": 132, + "p": 0.356, + "ci95": [ + 0.28, + 0.441 + ] + }, + "documents": { + "k": 1, + "n": 41, + "p": 0.024, + "ci95": [ + 0.004, + 0.126 + ] + }, + "small": { + "k": 3, + "n": 11, + "p": 0.273, + "ci95": [ + 0.097, + 0.566 + ] + }, + "formal_share_of_hits": { + "k": 6, + "n": 48, + "p": 0.125, + "ci95": [ + 0.059, + 0.247 + ] + }, + "examples": [ + "just-parallel-mechanism", + "just-telemetry-mechanism", + "ripgrep-pager-flag", + "appflowy-dark-mode-palette" + ] + }, + "G12 delegate|approve": { + "all": { + "k": 56, + "n": 173, + "p": 0.324, + "ci95": [ + 0.258, + 0.397 + ] + }, + "threads": { + "k": 44, + "n": 132, + "p": 0.333, + "ci95": [ + 0.259, + 0.417 + ] + }, + "documents": { + "k": 12, + "n": 41, + "p": 0.293, + "ci95": [ + 0.176, + 0.445 + ] + }, + "small": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "formal_share_of_hits": { + "k": 22, + "n": 56, + "p": 0.393, + "ci95": [ + 0.276, + 0.524 + ] + }, + "examples": [ + "appflowy-dark-mode-palette", + "astro-define-vars-scripts", + "astro-inline-script-stage-2", + "cargo-feature-doc-field" + ] + }, + "P6 consensus mechanics": { + "all": { + "k": 32, + "n": 173, + "p": 0.185, + "ci95": [ + 0.134, + 0.249 + ] + }, + "threads": { + "k": 16, + "n": 132, + "p": 0.121, + "ci95": [ + 0.076, + 0.188 + ] + }, + "documents": { + "k": 16, + "n": 41, + "p": 0.39, + "ci95": [ + 0.257, + 0.543 + ] + }, + "small": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "formal_share_of_hits": { + "k": 21, + "n": 32, + "p": 0.656, + "ci95": [ + 0.483, + 0.796 + ] + }, + "examples": [ + "crystal-io-first", + "eslint-native-ts-config-flag", + "excalidraw-new-drawing-button", + "go-remove-binary-only-packages" + ] + }, + "N reframe": { + "all": { + "k": 20, + "n": 173, + "p": 0.116, + "ci95": [ + 0.076, + 0.172 + ] + }, + "threads": { + "k": 14, + "n": 132, + "p": 0.106, + "ci95": [ + 0.064, + 0.17 + ] + }, + "documents": { + "k": 6, + "n": 41, + "p": 0.146, + "ci95": [ + 0.069, + 0.284 + ] + }, + "small": { + "k": 1, + "n": 11, + "p": 0.091, + "ci95": [ + 0.016, + 0.377 + ] + }, + "formal_share_of_hits": { + "k": 7, + "n": 20, + "p": 0.35, + "ci95": [ + 0.181, + 0.567 + ] + }, + "examples": [ + "just-telemetry-mechanism", + "excalidraw-handwritten-font", + "go-try-builtin", + "ha-addons-to-apps" + ] + }, + "G8 convention": { + "all": { + "k": 12, + "n": 173, + "p": 0.069, + "ci95": [ + 0.04, + 0.117 + ] + }, + "threads": { + "k": 3, + "n": 132, + "p": 0.023, + "ci95": [ + 0.008, + 0.065 + ] + }, + "documents": { + "k": 9, + "n": 41, + "p": 0.22, + "ci95": [ + 0.12, + 0.367 + ] + }, + "small": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "formal_share_of_hits": { + "k": 5, + "n": 12, + "p": 0.417, + "ci95": [ + 0.193, + 0.68 + ] + }, + "examples": [ + "excalidraw-zen-mode-entry-exit", + "prettier-final-newline", + "prettier-nested-ternary-format", + "eng-backstage-adr013-014-http-fetch-library" + ] + }, + "N visual comparison": { + "all": { + "k": 5, + "n": 173, + "p": 0.029, + "ci95": [ + 0.012, + 0.066 + ] + }, + "threads": { + "k": 4, + "n": 132, + "p": 0.03, + "ci95": [ + 0.012, + 0.075 + ] + }, + "documents": { + "k": 1, + "n": 41, + "p": 0.024, + "ci95": [ + 0.004, + 0.126 + ] + }, + "small": { + "k": 1, + "n": 11, + "p": 0.091, + "ci95": [ + 0.016, + 0.377 + ] + }, + "formal_share_of_hits": { + "k": 1, + "n": 5, + "p": 0.2, + "ci95": [ + 0.036, + 0.624 + ] + }, + "examples": [ + "fzf-preview-window", + "excalidraw-handwritten-font", + "vscode-completion-item-label-shape", + "zed-commit-log-graph" + ] + }, + "N rejected-content": { + "all": { + "k": 13, + "n": 173, + "p": 0.075, + "ci95": [ + 0.044, + 0.124 + ] + }, + "threads": { + "k": 3, + "n": 132, + "p": 0.023, + "ci95": [ + 0.008, + 0.065 + ] + }, + "documents": { + "k": 10, + "n": 41, + "p": 0.244, + "ci95": [ + 0.138, + 0.393 + ] + }, + "small": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "formal_share_of_hits": { + "k": 7, + "n": 13, + "p": 0.538, + "ci95": [ + 0.291, + 0.768 + ] + }, + "examples": [ + "deno-config-permissions", + "ha-entity-name-editing", + "ha-template-binary-sensor-delays-ui", + "eng-govuk-infrastructure-adr019-restructure-cicd" + ] + }, + "N consequences": { + "all": { + "k": 20, + "n": 173, + "p": 0.116, + "ci95": [ + 0.076, + 0.172 + ] + }, + "threads": { + "k": 5, + "n": 132, + "p": 0.038, + "ci95": [ + 0.016, + 0.086 + ] + }, + "documents": { + "k": 15, + "n": 41, + "p": 0.366, + "ci95": [ + 0.236, + 0.519 + ] + }, + "small": { + "k": 1, + "n": 11, + "p": 0.091, + "ci95": [ + 0.016, + 0.377 + ] + }, + "formal_share_of_hits": { + "k": 4, + "n": 20, + "p": 0.2, + "ci95": [ + 0.081, + 0.416 + ] + }, + "examples": [ + "fzf-default-opts-file", + "eslint-adopt-codemod", + "ha-zwave-driver-choice", + "home-assistant-lawn-mower-entity-model" + ] + }, + "G9 rank|partition": { + "all": { + "k": 10, + "n": 173, + "p": 0.058, + "ci95": [ + 0.032, + 0.103 + ] + }, + "threads": { + "k": 8, + "n": 132, + "p": 0.061, + "ci95": [ + 0.031, + 0.115 + ] + }, + "documents": { + "k": 2, + "n": 41, + "p": 0.049, + "ci95": [ + 0.013, + 0.161 + ] + }, + "small": { + "k": 2, + "n": 11, + "p": 0.182, + "ci95": [ + 0.051, + 0.477 + ] + }, + "formal_share_of_hits": { + "k": 5, + "n": 10, + "p": 0.5, + "ci95": [ + 0.237, + 0.763 + ] + }, + "examples": [ + "delta-light-dark-detection", + "fzf-preview-window", + "astro-scripts-only-when-rendered", + "eslint-formatting-rule-deprecation" + ] + }, + "G11 moot": { + "all": { + "k": 3, + "n": 173, + "p": 0.017, + "ci95": [ + 0.006, + 0.05 + ] + }, + "threads": { + "k": 1, + "n": 132, + "p": 0.008, + "ci95": [ + 0.001, + 0.042 + ] + }, + "documents": { + "k": 2, + "n": 41, + "p": 0.049, + "ci95": [ + 0.013, + 0.161 + ] + }, + "small": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "formal_share_of_hits": { + "k": 1, + "n": 3, + "p": 0.333, + "ci95": [ + 0.061, + 0.792 + ] + }, + "examples": [ + "mastodon-remote-local-timeline", + "eng-govuk-infrastructure-adr007-application-logging", + "tc39-2023-11-operator-overloading-withdrawn" + ] + }, + "C deadline": { + "all": { + "k": 3, + "n": 173, + "p": 0.017, + "ci95": [ + 0.006, + 0.05 + ] + }, + "threads": { + "k": 0, + "n": 132, + "p": 0.0, + "ci95": [ + 0, + 0.028 + ] + }, + "documents": { + "k": 3, + "n": 41, + "p": 0.073, + "ci95": [ + 0.025, + 0.194 + ] + }, + "small": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "formal_share_of_hits": { + "k": 3, + "n": 3, + "p": 1.0, + "ci95": [ + 0.438, + 1 + ] + }, + "examples": [ + "tc39-2020-04-weakrefs-finalizationregistry-cleanup", + "tc39-2021-08-realm-constructor-shadowrealm-name", + "tc39-2025-09-iterator-windows-undersized-parameter" + ] + }, + "X hidden history (corpus artefact)": { + "all": { + "k": 3, + "n": 173, + "p": 0.017, + "ci95": [ + 0.006, + 0.05 + ] + }, + "threads": { + "k": 3, + "n": 132, + "p": 0.023, + "ci95": [ + 0.008, + 0.065 + ] + }, + "documents": { + "k": 0, + "n": 41, + "p": 0.0, + "ci95": [ + 0, + 0.086 + ] + }, + "small": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "formal_share_of_hits": { + "k": 0, + "n": 3, + "p": 0.0, + "ci95": [ + 0, + 0.562 + ] + }, + "examples": [ + "eslint-formatting-rule-deprecation", + "typescript-node-esm-mode-shape", + "zed-git-panel-file-diff" + ] + }, + "P7 formal vote": { + "all": { + "k": 10, + "n": 173, + "p": 0.058, + "ci95": [ + 0.032, + 0.103 + ] + }, + "threads": { + "k": 3, + "n": 132, + "p": 0.023, + "ci95": [ + 0.008, + 0.065 + ] + }, + "documents": { + "k": 7, + "n": 41, + "p": 0.171, + "ci95": [ + 0.085, + 0.313 + ] + }, + "small": { + "k": 0, + "n": 11, + "p": 0.0, + "ci95": [ + 0.0, + 0.259 + ] + }, + "formal_share_of_hits": { + "k": 7, + "n": 10, + "p": 0.7, + "ci95": [ + 0.397, + 0.892 + ] + }, + "examples": [ + "node-primordials-error-path", + "node-yarn-vs-corepack", + "zed-git-panel-file-diff", + "eng-modernisation-platform-adr004-019-core-languages" + ] + } + }, + "flows": { + "all": { + "ask-teammate": { + "k": 118, + "n": 173, + "p": 0.682, + "ci95": [ + 0.609, + 0.747 + ] + }, + "build-pr": { + "k": 84, + "n": 173, + "p": 0.486, + "ci95": [ + 0.412, + 0.56 + ] + }, + "find": { + "k": 36, + "n": 173, + "p": 0.208, + "ci95": [ + 0.154, + 0.275 + ] + }, + "research-or-spike": { + "k": 64, + "n": 173, + "p": 0.37, + "ci95": [ + 0.302, + 0.444 + ] + }, + "reshape": { + "k": 41, + "n": 173, + "p": 0.237, + "ci95": [ + 0.18, + 0.306 + ] + }, + "revisit-reopen": { + "k": 78, + "n": 173, + "p": 0.451, + "ci95": [ + 0.379, + 0.525 + ] + }, + "settle": { + "k": 148, + "n": 173, + "p": 0.855, + "ci95": [ + 0.795, + 0.9 + ] + }, + "work-in-place": { + "k": 50, + "n": 173, + "p": 0.289, + "ci95": [ + 0.227, + 0.361 + ] + } + }, + "threads": { + "ask-teammate": { + "k": 99, + "n": 132, + "p": 0.75, + "ci95": [ + 0.67, + 0.816 + ] + }, + "build-pr": { + "k": 76, + "n": 132, + "p": 0.576, + "ci95": [ + 0.49, + 0.657 + ] + }, + "find": { + "k": 35, + "n": 132, + "p": 0.265, + "ci95": [ + 0.197, + 0.346 + ] + }, + "research-or-spike": { + "k": 54, + "n": 132, + "p": 0.409, + "ci95": [ + 0.329, + 0.494 + ] + }, + "reshape": { + "k": 38, + "n": 132, + "p": 0.288, + "ci95": [ + 0.218, + 0.37 + ] + }, + "revisit-reopen": { + "k": 55, + "n": 132, + "p": 0.417, + "ci95": [ + 0.336, + 0.502 + ] + }, + "settle": { + "k": 114, + "n": 132, + "p": 0.864, + "ci95": [ + 0.795, + 0.912 + ] + }, + "work-in-place": { + "k": 50, + "n": 132, + "p": 0.379, + "ci95": [ + 0.301, + 0.464 + ] + } + }, + "documents": { + "ask-teammate": { + "k": 19, + "n": 41, + "p": 0.463, + "ci95": [ + 0.321, + 0.613 + ] + }, + "build-pr": { + "k": 8, + "n": 41, + "p": 0.195, + "ci95": [ + 0.102, + 0.34 + ] + }, + "find": { + "k": 1, + "n": 41, + "p": 0.024, + "ci95": [ + 0.004, + 0.126 + ] + }, + "research-or-spike": { + "k": 10, + "n": 41, + "p": 0.244, + "ci95": [ + 0.138, + 0.393 + ] + }, + "reshape": { + "k": 3, + "n": 41, + "p": 0.073, + "ci95": [ + 0.025, + 0.194 + ] + }, + "revisit-reopen": { + "k": 23, + "n": 41, + "p": 0.561, + "ci95": [ + 0.41, + 0.701 + ] + }, + "settle": { + "k": 34, + "n": 41, + "p": 0.829, + "ci95": [ + 0.687, + 0.915 + ] + } + }, + "small": { + "ask-teammate": { + "k": 7, + "n": 11, + "p": 0.636, + "ci95": [ + 0.354, + 0.848 + ] + }, + "build-pr": { + "k": 9, + "n": 11, + "p": 0.818, + "ci95": [ + 0.523, + 0.949 + ] + }, + "find": { + "k": 3, + "n": 11, + "p": 0.273, + "ci95": [ + 0.097, + 0.566 + ] + }, + "research-or-spike": { + "k": 5, + "n": 11, + "p": 0.455, + "ci95": [ + 0.213, + 0.72 + ] + }, + "reshape": { + "k": 1, + "n": 11, + "p": 0.091, + "ci95": [ + 0.016, + 0.377 + ] + }, + "revisit-reopen": { + "k": 6, + "n": 11, + "p": 0.545, + "ci95": [ + 0.28, + 0.787 + ] + }, + "settle": { + "k": 9, + "n": 11, + "p": 0.818, + "ci95": [ + 0.523, + 0.949 + ] + }, + "work-in-place": { + "k": 2, + "n": 11, + "p": 0.182, + "ci95": [ + 0.051, + 0.477 + ] + } + } + } +} diff --git a/evals/datasets/decisions-v1/analysis/coverage-critique.md b/evals/datasets/decisions-v1/analysis/coverage-critique.md new file mode 100644 index 000000000..9bb630486 --- /dev/null +++ b/evals/datasets/decisions-v1/analysis/coverage-critique.md @@ -0,0 +1,237 @@ +# Coverage critique, rerun on decisions-v1 + +This reruns the dataset pass from [critique-coverage-data.md](critique-coverage-data.md) +on the combined decisions-v1 corpus. It tests the hypotheses H1–H12 from +[critique-coverage-theory.md](critique-coverage-theory.md) §5 again and re-ranks +the gaps. + +[`analyse_corpus.py`](analyse_corpus.py) produced every number below, and they +are all recorded in [`corpus-metrics.json`](corpus-metrics.json). Brackets are +Wilson 95% intervals. + +| Stratum | n | What it is | +| ----------- | --: | ------------------------------------------------------------------------------------------------------------------------------------------- | +| **All** | 173 | 132 thread records plus 41 document records | +| **Threads** | 132 | [`../decisions.jsonl`](../decisions.jsonl), refetched in full through the GitHub API | +| **Small** | 11 | Thread records in the small-team quota | +| Documents | 41 | [`../documents/decisions.jsonl`](../documents/decisions.jsonl): ADRs, KEPs, TSC minutes, TC39 notes, PEPs and Rust texts, shown for context | + +## Executive summary + +- **Verdicts:** + - **Supported:** H2, H5, H6, H7, H9, H10. + - **Refuted:** H1, narrowly, and H3 on the options half. + - **Inconclusive:** H8 in substance and H11. + - **Untestable:** H4. + - **Partly supported:** H12. Its vote half holds; its single-decider half is + supported only through the small-team proxy. + + Compared with v0, H1 moves from inconclusive to refuted and H11 from refuted to + inconclusive. Everything else keeps its verdict. +- **The end state is still the main gap, now with less noise.** Across all records: + - 61% [54–68] end in a composite answer: two or more options chosen, or one + adopted only in part. + - 50% [43–58] leave follow-up structure behind: child questions, "accepted in + principle", or obligations. + - 24% [18–31] are parked or idle without a verdict. + + The small-team stratum shows the same pattern, more strongly for the first: 73%, + 46% and 36%. The design has one answer and an Open/Settled binary. +- **"Decided elsewhere" is now the second most common gap: 53% [45–60].** In v0 it + ranked fifth, at 45%. Most of it is a meeting, call or chat, or a PR review that + decided while the thread waited, followed by what got built drifting from what + was agreed. In the small-team stratum it falls to 27%: solo maintainers decide + in the thread. +- **Crowd signal is real, not a fetch artefact.** v0 couldn't separate it from + webfetch summaries: 95% of its hits were non-git. On fully fetched threads it is + 36% [28–44], and 2% in documents. It is a product-thread pattern. +- **Shapes: keep four, and add no fifth.** + - Choice is 72% of records; tune 4%; mixed or none 9%. + - 82% of records carry a secondary shape, so mixing is coded as primary + + secondary. + - Check is primary in 7% of records, but primary or secondary in 32%. +- **For a solo builder, the small stratum points at a different order.** The top + gaps there are: + - composite answers (73%) + - follow-ups (46%) + - supersede or revert (46%) + - parked (36%) + + Delegation (0%), consensus mechanics (0%) and votes (0%) are absent. But the + stratum is only 11 records, with a median of 17 participants and 1,187 days per + decision. These are solo _maintainers_ fielding many users, not small groups. + +## 1. Method and biases + +**Records.** The 41 document records are the cloud session's v0 records that were +read from complete files (`git-file`). They are converted unchanged in substance, +and their provenance is in the [README](../README.md#document-records). Two +phrases that repeated eight or more words of their source were reworded. + +**Gap tags.** v0's hypotheses count gaps through a controlled tag vocabulary: one +tag per gap, from the theory pass §6 plus v0's additions. Document gaps keep the +v0 coder's tag. The 538 thread gaps had been named freely by their annotators, so +they were coded afterwards into the same vocabulary by two coders, one per half, +using a written brief with one calibration example per tag. The tags are in +[`thread-gap-tags.json`](thread-gap-tags.json). + +**Reliability.** A third coder blind-recoded 60 randomly sampled document gaps, +using a calibration set that excluded them. Agreement with the v0 coder was 70%, +Cohen's κ = 0.68 across 29 categories (chance agreement 0.07), which is +substantial. The disagreements are scattered single confusions, such as +provisional→conditional and blocking→delegate, not a systematic drift. The data +is in [`tag-reliability.json`](tag-reliability.json). + +| Bias | Evidence | Handling | +| ---------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | +| Genre effect on some tags | `consequences` 37% in documents vs 4% in threads; `rejected-content` 24% vs 2%; `convention` 22% vs 2%. The blind recode reproduced these tags at the v0 coder's rate, so this isn't coder drift. ADRs and PEPs have Consequences and Rejected Alternatives sections; thread annotators rarely wrote gaps about them. | Reported per stratum. Read thread rates for these tags as lower bounds. | +| Thread coders saw the annotator's cluster name | Each thread gap begins with its annotator's free cluster name; the document recode had no prefix. | The blind recode measures the coding, not the hint. The hint can only pull thread tags towards the annotator's view. | +| Small-team ≠ small group | Small: median 17 participants, median 1,187 days, 9/11 with a single named decider. `participants.count` counts every human commenter, so only 3 records in the whole corpus have ≤3 participants. | The small stratum stands in for the persona. H12's ≤3 test is untestable. | +| Debated-only selection | Every thread was screened for a genuine debate. The PR-debate quota is PR-review threads by construction. | Direction, not size. H11 is reported per stratum. | +| Saturated fields | `process.subQuestions` non-empty in 99%; `criteria` non-empty in 100% of choice records. | Neither is used as a signal. H6 uses tags; H3's criteria half is untestable. | +| Inconsistent coding | `confidence: stated` is 73% in threads vs 45% in documents. Neither annotation guide defined "stated". | H8's confidence half isn't used for the verdict. | +| Option order | Neither guide asked for options in proposal order. | H4 is untestable. | +| Defaulted field | v0 had no `decidedOffPlatform`, so document records default to `false`. | Not used. "Decided elsewhere" is measured through the `venue-split` tag instead. | +| One annotator per record | Shapes and outcomes were coded once. v0's shape κ was 0.62 on its own sample; it wasn't re-estimated for threads. | Shape-based results carry moderate confidence. | + +## 2. Hypotheses + +The thresholds come from theory §5. "Small" has n = 11, so its intervals are wide. + +| H | Prediction | All | Threads | Small | Verdict | Conf. | v0 | +| --- | --------------------------------------------------------------------------- | ------------------------------------------------------------------ | --------------- | ---------------------------------------- | -------------------------------------------------------------------------- | ------------------- | ------------------ | +| H1 | mixed or none ≥15%; falsified <10% | 16/173 **9%** [6–14] | 8% [5–14] | 0/11 [0–26] | **REFUTED** (narrowly) | Low | Inconclusive (11%) | +| H2 | choice modal; tune ≤10% | choice **72%**, tune **4%** [2–8] | 70% / 5% | 73% / 9% | **SUPPORTED** | High for the corpus | Supported | +| H3 | median options ≤3; criteria ≤50% | median **5**; criteria 100% | median 6 | median 5.5 | **REFUTED** (options); criteria untestable | Moderate | Refuted | +| H4 | options[0] wins ≥60% | 57/148 **38%** [31–46] | 36% | 5/10 | **UNTESTABLE** | — | Untestable | +| H5 | parked ≥15%; deferred ≥ abandoned | **24%** [18–31]; 23 deferred vs 3 abandoned | 23%; 13 vs 3 | 36%; 0 vs 0 | **SUPPORTED** | Moderate | Supported | +| H6 | structure ≥20%; rfc or proposal ≥40% | **65%** [58–72]; rfc or proposal **84%** [67–93] | 61%; 82% | 46%; n/a | **SUPPORTED** | High for direction | Supported | +| H7 | non-check uses research or a spike ≥30% | strict **70%** [62–76]; without implementation PRs **35%** [28–42] | 79%; 35% | 90%; 40% | **SUPPORTED** | Moderate | Supported | +| H8 | reversibility stated ≥25%; confidence ≤15% | reversibility **40%** [33–49]; confidence **67%** [59–74] | 44% / 73% | 20% / 50% | **REFUTED as specified; inconclusive in substance** | Low | Inconclusive | +| H9 | dissent survives ≥25% (decided, ≥3 participants) | 116/132 **88%** [81–92] | 89% | 6/10 **60%** [31–83] | **SUPPORTED** | Moderate | Supported | +| H10 | reopened or superseded ≥10%; ≥30% of those tagged | **29%** [23–36]; **70%** of those | 26%; 74% | 46%; 5/5 | **SUPPORTED** | Moderate | Supported | +| H11 | PR source ∧ implementation PR ≥20% | 29/173 **17%** [12–23] | **22%** [16–30] | 1/11 **9%** | **INCONCLUSIVE** | Low | Refuted (0/82) | +| H12 | ≤3 participants: single decider ≥75%; votes or polls ≤10% outside standards | ≤3: n = 3 (2/3); votes or polls **10%** [6–15] (non-formal 7%) | 3 (2/3); 8% | single decider **82%** [52–95]; votes 0% | **Votes half SUPPORTED; ≤3 half untestable, supported by the small proxy** | Moderate | Inconclusive | + +Notes on each hypothesis: + +- **H1.** The point estimate is under the 10% falsification line, but the interval + reaches 14%, so this is a narrow refutation. 82% of records carry a secondary + shape. Every mixed record is a bundle of coupled sub-choices: a + parent-and-children structure problem, as v0 found. +- **H2.** Tune is 13% [6–24] of UI and product records as a primary shape, and 44% + [32–58] as primary or secondary. Documents contain no tune records. +- **H3.** Public debate piles up variants: threads have a median of 6 options. In + 55% [46–63] of choice records more than one option is chosen or one is adopted in + part, and 56% list at least one option deferred. +- **H4.** Point estimates would refute it, but option order is not chronological. +- **H5.** The raw `deferred` flag (38%) overstates it, because 43 decided records + deferred only part of the question. The test uses undecided ∧ deferred, or a + `deferred` or `stale-idle` tag. Small's 36% comes entirely from tags: stale and + idle threads. +- **H6.** `follow_up_structure` (nested, staged or consequences tags) is 50% [43–58] + for all records, 45% for threads, 46% for small and 68% for documents. +- **H7.** Thread spikes are dominated by `implementation-pr` (59 of 121), so the + second figure excludes it. It still clears 30% in every stratum, but its interval + dips below. Spike mix across all records: implementation PR 64, none 53, prototype + 35, feature flag 5, experiment 4. +- **H8.** Confidence is coded as "the decision was stated firmly", not as a + calibrated confidence. Reversibility is stated in 40% of records, which is the + part the design can use. +- **H9.** Debated-only selection inflates dissent. The direction is safe, and in the + small stratum dissent still survives in 6 of 10 decisions. +- **H10.** The collection sought reversals, so the size is inflated, but small + maintainers revisit most often. +- **H11.** Threads pass only because PR-review debates were a quota. Documents have + 0/41 and small has 1/11. An implementation PR from any source appears in 41% + [34–48] of all records. This still doesn't bear on G3, which concerns undebated + agent calls. +- **H12.** Single named decider: 33% of all records, 41% of threads, 7% of + documents. Team or consensus decides 51% of all records. + +## 3. Gaps re-ranked + +The share is the proportion of records with at least one gap in the group. Groups +follow theory §3 (G-ids) and v0's added groups (N, P, C). Examples are decision +IDs. "Formal" is the share of hits from formal processes: Rust, Go and Python +proposals, TC39 and thread cases in the formal-process quota. + +| # | Gap (tags) | All | Threads | Small | Documents | Formal share | Examples | Solo builder? | +| -- | ------------------------------------------------------------------ | --------------- | --------------- | --------------- | --------- | ------------ | -------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | +| 1 | **Composite answer** (≥2 chosen or partly/merged) | **61%** [54–68] | 63% [54–71] | **73%** [43–90] | 56% | 24% | `bat-builtin-pager-approach`, `delta-light-dark-detection`, `fzf-preview-window` | **Yes** | +| 2 | **Decided elsewhere** (`venue-split`, `implicit`, `ratify`) | **53%** [45–60] | 55% [47–64] | 27% [10–57] | 44% | 29% | `bat-builtin-pager-approach`, `fzf-default-opts-file`, `vite-node-worker-api` | **Partly.** For a solo builder it's Chat, an agent in code, or another document. | +| 3 | **Follow-ups** (`nested`, `staged`, `consequences`) | **50%** [43–58] | 45% [36–53] | 46% [21–72] | 68% | 38% | `delta-light-dark-detection`, `fzf-default-opts-file`, `rfcs-derive-from-direction` | **Yes** | +| 4 | **Coupled / blocking** (`coupled`, `blocking`) | **38%** [31–46] | 39% [32–48] | 18% [5–48] | 34% | 30% | `just-f-string-syntax`, `ripgrep-long-line-handling`, `go-loopvar-per-iteration` | Yes, but #3 covers the parent/child cases | +| 5 | **Supersede / revert** (`supersede`, `revert`) | **32%** [25–39] | 33% [25–41] | **46%** [21–72] | 29% | 29% | `ripgrep-hyperlink-support`, `pnpm-pre-post-scripts`, `zoxide-fish-cd-call` | **Yes**, once code has shipped on the old answer | +| 6 | **Delegate / approve** (`delegate`, `approve`) | 32% [26–40] | 33% [26–42] | **0%** [0–26] | 29% | 39% | `vite-eager-browserhash-merge`, `deno-config-permissions` | **Teams only** | +| 7 | **Evidence beyond check** (`evidence-not-check`) | 28% [22–36] | 30% [23–39] | 27% [10–57] | 22% | 20% | `just-parallel-mechanism`, `go-loopvar-per-iteration`, `excalidraw-binding-default` | **Yes** | +| 8 | **Crowd signal** (`crowd-signal`) | 28% [22–35] | **36%** [28–44] | 27% [10–57] | 2% | 12% | `pnpm-pm-strict-default`, `just-telemetry-mechanism`, `go-try-builtin` | **Partly:** builders with users | +| 9 | **Parked / idle** (`deferred`, `stale-idle`, undecided ∧ deferred) | 24% [18–31] | 23% [16–31] | **36%** [15–65] | 29% | 26% | `just-parallel-mechanism`, `delta-light-dark-detection`, `home-assistant-last-changed-persistence` | **Yes** | +| 10 | Team settle / open objection (`consensus-mechanics`) | 18% [13–25] | 12% | 0% | 39% | **66%** | `crystal-io-first`, `eslint-native-ts-config-flag` | Teams only | +| 11 | Provisional / conditional (`provisional`, `conditional`) | 14% [10–20] | 8% | 9% | 34% | 33% | `bat-debian-batcat-name`, `eslint-native-ts-config-flag`, backstage ADR014 | Yes; thread rate is a lower bound (genre) | +| 12 | Reframe (`reframe`) | 12% [8–17] | 11% | 9% | 15% | 35% | `go-try-builtin`, `excalidraw-handwritten-font` | Yes, when research disproves the premise | +| 12 | Consequences, as a tag | 12% [8–17] | 4% | 9% | 37% | 20% | ADR records | Yes; genre effect | +| 14 | External authority (`external-authority`) | 10% [6–15] | 6% | 18% | 22% | 24% | `ripgrep-hyperlink-support`, `bat-debian-batcat-name` | Yes | +| 15 | Rejected-option content (`rejected-content`) | 8% [4–12] | 2% | 0% | 24% | 54% | PEP and ADR records | Partly; genre effect | +| 16 | Convention / scope (`convention`) | 7% [4–12] | 2% | 0% | 22% | 42% | `prettier-final-newline`, MoJ ADR019 | Yes for agents; the corpus can't measure it | +| 17 | Rank / partition (`rank`, `partition`) | 6% [3–10] | 6% | **18%** | 5% | 50% | `fzf-preview-window`, `delta-light-dark-detection` | Partly; covered by #1 | +| 17 | Formal vote (`vote`) | 6% [3–10] | 2% | 0% | 17% | 70% | `node-yarn-vs-corepack`, PEP 8016 | No | +| 19 | Visual comparison | 3% [1–7] | 3% | 9% | 2% | 20% | `excalidraw-handwritten-font` | Partly: UI work | +| 20 | Moot, deadline, hidden history | 2% each | ≤2% | 0% | ≤7% | — | — | Rare. Hidden history falls from v0's 12% to 2%, which confirms it was a webfetch artefact. | + +**What changed from v0's ranking:** + +- Decided elsewhere rises from #5 to #2. +- Supersede/revert rises from #8 to #5, and to joint #2 in the small stratum. +- Crowd signal becomes credible. +- Delegate/approve and consensus mechanics are confirmed as team gaps, absent from + the small stratum. +- Provisional/conditional falls from #7 to #11 overall. It stays high in documents, + where ADRs and PEPs state revisit conditions. + +## 4. What to do + +This follows v0's three-way split, updated with the new evidence. + +**(a) Make now.** The evidence is strong and holds in the small stratum. + +1. **Each choice row takes a fate:** chosen, partly, later or not this. The settled + sentence is composed from the chosen and partly rows. Composite answers are #1 + overall and 73% of small. +2. **Follow-ups on Settle, and "settled in principle":** child questions linked to + the parent, plus stated obligations (#3). +3. **Record a decision made elsewhere,** with a source link and captured evidence + (#2). For a solo builder this is the decision taken in Chat or by an agent in + code. +4. **Reopen keeps the old answer as Superseded,** and Built into shows revert PRs + (#5). +5. **A Parked state with a revisit trigger** (#9, 36% of small). +6. **Research and Spike on every shape,** per option on choice (#7). + +**(b) Design for, but defer:** + +- coupled and blocking links (#4) +- user-signal evidence, meaning reaction counts as evidence rather than a vote (#8) +- a "Driven by" field for outside authority (#14) +- reframe history (#12) +- convention scope (#16) +- image options (#19) + +**(c) Out of scope for this persona:** + +- votes and tallies +- unanimity and objection mechanics +- delegation rosters + +None appears in the small stratum, and 66–70% of their hits are formal processes. + +## 5. What this corpus still can't tell us + +- **G1 (ratify) and G3 (undebated agent calls).** Every record was selected for + debate. +- **A small _group_.** The small stratum is solo maintainers with many users. The + persona needs Chopin's own planning sessions, or private repos. +- **Proposal order and calibrated confidence (H4, H8).** Both need an annotation + rule before the next collection. +- **Shape reliability on threads.** A double-coded sample, as v0 did for its + records, would settle H1's narrow margin. diff --git a/evals/datasets/decisions-v1/analysis/critique-coverage-data.md b/evals/datasets/decisions-v1/analysis/critique-coverage-data.md new file mode 100644 index 000000000..fa446099e --- /dev/null +++ b/evals/datasets/decisions-v1/analysis/critique-coverage-data.md @@ -0,0 +1,184 @@ +> **Provenance.** Written in an earlier cloud session against corpus v0 (82 records, including webfetch summaries). Paths such as `review/…`, `decisions/…` and `SPEC.md` refer to that session's scratchpad. Kept unchanged as the basis for [coverage-critique.md](coverage-critique.md), which reruns the tests on decisions-v1. + +# Critique: coverage, dataset pass + +Tests review/critique-coverage-theory.md §5 (H1–H12) and §3 (G1–G12) against decisions/corpus-v0-tagged.jsonl (82 records). Script: review/analyse_corpus.py (`python3 -I`). Every number below is in review/corpus-metrics.json. Brackets are Wilson 95% intervals. **Persona** = `personaRelevance` high + medium (n = 45); **Low** = low (n = 37); **Git-P** = git-file records within Persona (n = 23). + +## Executive summary + +- **The corpus is committee-heavy, and "persona-relevant" doesn't mean "small group".** Where participant counts are known (55/82), the median is 11. Only 3 records have ≤3 participants, and 52% come from formal-process collectors (TC39, Rust, Go/Python). The persona stratum has a median of 10 participants. It is persona-relevant by _content_, not by group size, and it is mostly product-UI threads (all webfetch) plus ADRs. Read every rate as "teams debating in public", which is an upper bound for the coordination gaps and only a hint for a solo builder. +- **Verdicts.** SUPPORTED: H2, H5, H6, H7, H9, H10. REFUTED: H3 (on the options half), H11. INCONCLUSIVE: H1, H4, H8, H12. +- **The main coverage gap is the end state, not the shapes.** Real decisions rarely end as one clean answer. In the Persona stratum: + - 69% [54–80] leave follow-up structure behind: sub-questions, "accepted in principle, details later", or stated consequences and obligations. + - 42% [29–57] end in a composite answer: two or more options chosen, or one adopted only in part. + - 29% [18–43] are parked or closed without a verdict. + + The design has one answer and an Open/Settled binary. All three findings hold on git-file records: 69%, 50% and 29%. +- **Shapes: keep four, and add no fifth.** + - Choice is 67% of records and tune is 5% (24% in UI and product). + - All 9 "mixed" records are bundles of coupled sub-choices. That's a structure problem (a parent question with children), not a missing shape. + - Check is the primary shape in only 6% of records, but appears as primary or secondary in 32%. It behaves like evidence attached to other questions. + - Shape κ is 0.618, with chance agreement of 0.60. Both recoding disagreements moved _to_ choice. Choice's dominance is partly an effect of how the corpus was selected and of the schema. +- **(a) Make now:** + 1. Follow-ups on Settle. + 2. A fate for each option in choice. + 3. A Parked state with a revisit trigger. + 4. Research and Spike on every shape. + 5. Reopen keeps the old answer as superseded. + 6. A "Driven by" field for outside authorities. + 7. Recording a decision after the fact, with a source link. +- **(b) Defer, but design for:** handing a question to another owner, team settles with an open-objection warning, coupled and blocking links, user-signal evidence, reframe history, convention scope, and image options. +- **(c) Out of scope:** votes and tallies, unanimity and blocking mechanics, stage gates, FCP and review boards, escalation, moderation, and release-train deadlines. +- **Over-built:** + - Tune's bespoke preview: no git-file record is tune-shaped. + - Tallies: votes or polls appear in 8.5% of records. + - Escalation: present in 7% of records. + - Takes are _not_ over-built: dissent survives in 76% of decided records with ≥3 participants. +- **What the corpus can't test:** + - G1 (ratify Chopin's proposal) and G3 (calls agents make in code), because undebated decisions are excluded by construction. + - H4, because option order isn't chronological. + - Confidence, because it was coded inconsistently: 0% for the ADR collector against 75% for TC39. + +--- + +## 1. Biases and how they're handled + +| Bias | Evidence in this corpus | Handling | +| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| Committee-heavy | Median 11 participants. 3/82 have ≤3 participants, and none of the 3 has a single decider. Team or committee deciders: 66%. | Stratified by `personaRelevance`. H12's ≤3 stratum is untestable. | +| Persona stratum is confounded | Persona = product-UI 16, ADRs 12, chopin-v2 6, other 11. High = product-UI 11 of 17, mostly webfetch. | Git-P column. Gaps that vanish on git-file are flagged. | +| Fidelity | 34/82 are webfetch or mixed, 28 of them truncated. Git-file contains **no** product-UI records. | Git-file column on every metric. Crowd-signal (95% non-git), image comparison (83%) and hidden history (100%) are fetch- or collector-specific. | +| Option order | No record states chronological order. First option chosen: ADR collector 0/12, TC39 10/12. | H4 marked untestable. | +| Debated-only selection | Debate inflates dissent, venue-split, consensus and reopening. The handoff asked collectors for reversals, and ADR chains were collected as sets (`adr012-021-036-048`). | Persona and Low reported side by side. H9 and H10 magnitudes discounted. | +| **Saturated fields (found here)** | `process.subQuestions` is non-empty in 81/82 records. `criteria` is non-empty in 55/55 choice records. | Neither field is used as a signal. H6 uses tags only. H3's criteria half is untestable. | +| **Inconsistent coding (found here)** | `confidence: stated` ranges from 0/12 for the ADR collector to 9/12 for TC39. `process.deferred` is true on 20 _decided_ records, meaning part of the decision was deferred. | H8 confidence half is not used. H5 uses undecided ∧ deferred. | +| Tagger priming | The gap tags use the theory's own vocabulary, in a single pass with no double-coding of tags. | Treat the size of each tag rate as low confidence, and the ranking between tags as moderate confidence. | + +## 2. Hypotheses + +| H | Prediction | All | Persona | Git-file | Verdict | Conf. | Caveat | +| --- | ---------------------------------------------------------------------------------- | ----------------------------------------------------------------- | ------------------ | -------------- | ---------------------------------------------- | ---------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| H1 | mixed or none ≥15% (falsified <10%) | 9/82 **11%** [6–20] | 2/45 4% [1–15] | 5/48 10% | **INCONCLUSIVE** | Low | Falls between the thresholds. 67% of records carry a `secondaryShape`, so mixing is real but gets coded as primary + secondary. Shape κ is borderline. | +| H2 | choice is modal; tune ≤10% | choice **67%** [56–76], tune **5%** [2–12] | 71% / 9% | 75% / 0% | **SUPPORTED** | High for the corpus, low for the persona | Tune in ui-ux and product-behaviour: 4/17, 24% [10–47], as predicted. Every tune record is a webfetch product-UI thread. The selection rule ("≥2 options weighed") pushes towards choice. | +| H3 | median options ≤3; criteria ≤50% | median **5**; criteria **100%** | median 4 | Git-P median 3 | **REFUTED** (options); criteria **untestable** | Low–moderate | Public threads pile up variants and the status quo. Coders always extracted criteria, so 100% says nothing about whether the sources made criteria explicit. | +| H4 | options[0] wins ≥60% | 35/62 **56%** [44–68] | 51% | 51% | **INCONCLUSIVE (untestable)** | — | The rate depends on each collector's convention: ADRs list the winner late, proposal processes list the champion's proposal first. Without ADRs it's 70%. Don't force it. | +| H5 | deferred ≥15%; deferred ≥ abandoned | whole-question parked **21%** [13–31]; 17 deferred vs 3 abandoned | 16% [8–29]; 7 vs 3 | 23%; 11 vs 0 | **SUPPORTED** | Moderate | The raw flag (45%) overstates it, because it includes partial deferral. In 40% of choice records at least one option has the fate `deferred`. | +| H6 | structure ≥20%; rfc or proposal ≥40% | tags only **77%** [67–85]; rfc or proposal 15/17 | 76% | 79% | **SUPPORTED** | High for direction, low for size | `subQuestions` is saturated, so tags only. Priming and selection inflate the rate. Even half of it passes. | +| H7 | non-check uses research or a spike ≥30% | broad 79%; **strict 40%** [30–51] | strict 43% | strict 41% | **SUPPORTED** | Moderate–high | "Strict" excludes `research: links`. Choice+check is the commonest primary+secondary pair (19/82). | +| H8 | reversibility stated ≥25%; confidence ≤15% | reversibility **34%** [23–46]; confidence 55% | 34% / 34% | 26% / 51% | **INCONCLUSIVE** | Low | Reversibility half: supported. Confidence half: refuted on paper, but the field was coded as "the decision was stated firmly" by the formal collectors and strictly by the ADR collector. | +| H9 | dissent survives ≥25% (decided, ≥3 participants) | 29/38 **76%** [61–87] | 80% | 79% | **SUPPORTED** | Moderate | Debated-only selection inflates dissent. The direction is safe. | +| H10 | reopened or superseded ≥10%; ≥30% of those tagged supersede or revert | **29%** [20–40]; 54% of those | 31%; 50% | 31%; 60% | **SUPPORTED** | Moderate on direction, low on size | Collectors deliberately sought reversals and ADR chains. | +| H11 | PR source ∧ implementation-pr ≥20% | **0/82** [0–4] | 0/45 | 0/48 | **REFUTED** (as specified) | Low relevance | Only 11 records have a PR source, and the planned 20 PR-review debates are absent. Implementation PR from any source: 12%. This doesn't bear on G3. | +| H12 | ≤3 participants: single decider ≥75%; votes or polls ≤10% outside standards bodies | ≤3: n = 3, 0/3 single; votes or polls outside TC39 **8%** [3–16] | 5% | 12% | **INCONCLUSIVE** | — | The first half is untestable. The second half is supported. A single named decider appears in only 13% of all records, which reflects who the corpus samples. | + +## 3. Gaps re-ranked + +Ranked by Persona frequency × confidence × relevance to a solo builder. G-ids are from the theory pass. Data tags are in `code`. The percentage is the share of records with ≥1 such gap. + +| # | Gap (theory id; data tags) | All | Persona | Low | Git-P | Examples | Solo builder? | Smallest design change | Conf. | +| -- | ------------------------------------------------------------------------------------------------- | ------------------------ | -------- | -------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------- | +| 1 | **Follow-ups: nested, staged, consequences** (G7; `nested` 35%, `staged` 30%, `consequences` 30%) | 67% | 69% | 65% | 78% | [backstage ADR014](https://github.com/backstage/backstage/blob/master/docs/architecture-decisions/adr014-use-fetch.md), [astro #655](https://github.com/withastro/roadmap/discussions/655), [govuk ADR007](https://github.com/alphagov/govuk-infrastructure/blob/main/docs/architecture/decisions/0007-use-fluentbit-elasticsearch-and-kibana-for-application-logs.md) | **Yes.** Build agents need the obligations and the open children. | **Field + link.** Settle takes "Follow-ups": consequences or obligations (text), plus child questions linked to the parent. A parent can be settled while its children stay open ("settled in principle"). | Moderate–high. `consequences` is 42% in Persona against 16% in Low, partly because the ADR format has a Consequences section. | +| 2 | **Composite answer** (G9 recast; `rejected-content`) | 45% | 42% | 49% | 48% | [pandas #54154](https://github.com/pandas-dev/pandas/issues/54154), [govuk ADR019](https://github.com/alphagov/govuk-infrastructure/blob/main/docs/architecture/decisions/0019-restructure-cicd.md), [vscode #334285](https://github.com/microsoft/vscode/issues/334285) | **Yes** | **Shape tweak.** Each choice row takes a fate (chosen / partly / later / not this), with an optional one-line reason. The settled sentence is composed from the chosen and partly rows. This also covers rank and partition. | Moderate–high. Based on option fates, which need less judgement than gap tags. Option lists are inflated by public debate. | +| 3 | **Parked / closed without a verdict** (G2, G11; `deferred`, `stale-idle`, `moot`) | 33% | 29% | 38% | 17% | [biome #3339](https://github.com/biomejs/biome/issues/3339), [astro #655](https://github.com/withastro/roadmap/discussions/655), [excalidraw #1957](https://github.com/excalidraw/excalidraw/issues/1957) | **Yes** | **State.** Parked: neutral, not counted in the capsule, doesn't block Build plan, with an optional "waiting on" and a revisit trigger. "Close without deciding", with a reason, replaces Discard for moot questions and keeps a one-line record. | High that it's real; moderate on size. Git-P is only 17%, and stale threads are a product-UI pattern. | +| 4 | **Evidence beyond check** (G6; `evidence-not-check`) | 26% (tag); 40% strict H7 | 22%; 43% | 30%; 37% | 13%; 32% | [govuk ADR018](https://github.com/alphagov/govuk-infrastructure/blob/main/docs/architecture/decisions/0018-use-concourse-ci.md) (parallel spikes per option), [mastodon #12753](https://github.com/mastodon/mastodon/issues/12753), [HA arch #1340](https://github.com/home-assistant/architecture/discussions/1340) | **Yes.** Agents make spiking each option cheap. | **Shape tweak.** Research and Spike on choice (per option) and on explore. A check can attach to a choice cell as a claim. | Moderate–high | +| 5 | **Decided elsewhere / after the fact** (G3 proxy; `venue-split`, `implicit`, `ratify`) | 45%; 9% | 36%; 13% | 57%; 3% | 22%; 13% | [rust roadmap](https://github.com/rust-lang/lang-team/blob/c45345e32e81e273aab6c805b0d3cd7b1a666045/design-meeting-minutes/2022-03-09-lang-roadmap.md), [outline #3326](https://github.com/outline/outline/discussions/3326), [govuk ADR019](https://github.com/alphagov/govuk-infrastructure/blob/main/docs/architecture/decisions/0019-restructure-cicd.md) | **Partly.** The solo version is a decision made in Chat, by an agent in code, or in another document. | **Entry point + field.** "Record a decision" from a Chat message or PR creates a settled question. The record gets a "Decided in" source link, and external evidence is captured, not just linked. | Moderate. `venue-split` is 60% formal collectors, so the solo analogue is an inference. | +| 6 | **External driver / authority** (P8; `external-authority`) | 24% | 33% | 14% | 22% | [cloud-platform ADR040](https://github.com/ministryofjustice/cloud-platform/blob/main/architecture-decision-record/040-use-github-actions.md), [govuk ADR018](https://github.com/alphagov/govuk-infrastructure/blob/main/docs/architecture/decisions/0018-use-concourse-ci.md), [excalidraw #2945](https://github.com/excalidraw/excalidraw/issues/2945) | **Yes:** platform rules, clients, upstream projects, audits | **Field.** "Driven by" (person outside Chopin / rule / upstream decision), shown in the facts row beside Settled by. | Moderate | +| 7 | **Provisional / conditional, with a revisit trigger** (G5; `provisional`, `conditional`) | 29% | 24% | 35% | 26% | [backstage ADR014](https://github.com/backstage/backstage/blob/master/docs/architecture-decisions/adr014-use-fetch.md), [govuk ADR007](https://github.com/alphagov/govuk-infrastructure/blob/main/docs/architecture/decisions/0007-use-fluentbit-elasticsearch-and-kibana-for-application-logs.md), [zed #26344](https://github.com/zed-industries/zed/discussions/26344) | **Yes** | **Field.** Optional "Revisit if…" on Settle, shared with Parked. Split reversibility out of the confidence string. Reversibility is stated in 34% of records. | Moderate | +| 8 | **Supersede / revert** (G10; `supersede`, `revert`) | 24% | 24% | 24% | 26% | [backstage ADR014](https://github.com/backstage/backstage/blob/master/docs/architecture-decisions/adr014-use-fetch.md), [cloud-platform ADR019](https://github.com/ministryofjustice/cloud-platform/blob/main/architecture-decision-record/019-Shared-Ingress-Controllers.md), [mastodon #12753](https://github.com/mastodon/mastodon/issues/12753) | **Partly.** It matters once code has shipped on the old answer. | **Link.** Reopen archives the previous answer as "Superseded" (still readable, with a forward link), and Built into shows revert PRs. | Moderate on direction, low on size (selection) | +| 9 | **Coupled / blocking** (G4; `coupled`, `blocking`) | 32% | 29% | 35% | 30% | [eslint RFC #134](https://github.com/eslint/rfcs/pull/134), [govuk ADR018](https://github.com/alphagov/govuk-infrastructure/blob/main/docs/architecture/decisions/0018-use-concourse-ci.md), [node TSC ncrypto](https://github.com/nodejs/TSC/blob/main/meetings/2026-02-11.md) | **Yes,** but #1 covers the parent/child cases | **Link.** "Depends on" / "affects". Settling A marks B "check again". | Moderate. 62% of hits are from formal collectors. | +| 10 | **Crowd signal** (P10; `crowd-signal`) | 24% | 40% | 5% | **0%** | [excalidraw #1957](https://github.com/excalidraw/excalidraw/issues/1957), [HA arch #717](https://github.com/home-assistant/architecture/discussions/717), [go #32437](https://github.com/golang/go/issues/32437) | **Partly:** builders with users | **Evidence type.** "User signal" (feedback links, reaction counts) in What we knew. | Low. 95% of hits are webfetch product-UI, so collector and fetch method are confounded. | +| 11 | **Delegate / approve** (G12; `delegate`, `approve`) | 27% | 20% | 35% | 17% | [eslint RFC #134](https://github.com/eslint/rfcs/pull/134), [outline #3326](https://github.com/outline/outline/discussions/3326), [go #32437](https://github.com/golang/go/issues/32437) | **Teams only** | **Role.** Decider ≠ owner, and the owner can hand over. Approver later. | Moderate. 68% of hits are from formal collectors. | +| 12 | **Team settle / open objection** (P6; `consensus-mechanics`) | 33% | 18% | 51% | — | [deno #25575](https://github.com/denoland/deno/issues/25575), [pandas #54154](https://github.com/pandas-dev/pandas/issues/54154), [eslint RFC #134](https://github.com/eslint/rfcs/pull/134) | **Teams only.** 0/17 of the High stratum. | **Field tweak.** Settled by can be "the team". Settle warns about an unanswered Objects take. Full mechanics are out of scope. | Moderate. 82% of hits are from formal collectors. | +| 13 | **Reframe / premise drift** (`reframe`) | 18% | 20% | 16% | 13% | [biome #3339](https://github.com/biomejs/biome/issues/3339), [zed #26344](https://github.com/zed-industries/zed/discussions/26344), [node TSC ncrypto](https://github.com/nodejs/TSC/blob/main/meetings/2026-02-11.md) | **Yes,** when research disproves the premise | **Timeline event + F8.** Log rewordings in How it moved, and let Chopin propose a reshape when findings contradict the premise. | Low–moderate | +| 14 | **Convention / scope** (G8; `convention`) | 15% | 13% | 16% | 17% | [MoJ ADR019](https://github.com/ministryofjustice/modernisation-platform/blob/main/architecture-decision-record/0019-use-bash-go-as-core-languages.md), [backstage ADR014](https://github.com/backstage/backstage/blob/master/docs/architecture-decisions/adr014-use-fetch.md), [node TSC macOS tier 2](https://github.com/nodejs/TSC/blob/main/meetings/2026-05-06.md) | **Yes for agents.** The corpus can't measure it. | **Field.** Scope: this plan / this repo. Repo-scoped records are exported to agent instructions. | Low (n = 12, partly untestable) | +| 15 | **Image or mock-up options** (`visual-comparison`) | 7% | 13% | 0% | 4% | [excalidraw #2945](https://github.com/excalidraw/excalidraw/issues/2945), [HA frontend #8905](https://github.com/home-assistant/frontend/discussions/8905), [PEP 602](https://peps.python.org/pep-0602/) | **Partly:** UI work | **Shape tweak.** A choice row can carry an image. | Low | +| 16 | Rank / partition (G9 as stated) | 4% | 4% | 3% | 4% | [rust roadmap](https://github.com/rust-lang/lang-team/blob/c45345e32e81e273aab6c805b0d3cd7b1a666045/design-meeting-minutes/2022-03-09-lang-roadmap.md), [HA frontend #8905](https://github.com/home-assistant/frontend/discussions/8905) | Rare | Covered by #2. **No new shape.** | Moderate | +| 17 | Ratify the proposal (G1; `ratify`) | 1% | 2% | 0% | 0% | [vscode #334285](https://github.com/microsoft/vscode/issues/334285) | Yes, in theory | Keep the theory's recommendation: an answer-origin field and Accept. **The corpus can't test it.** | None from the data | +| 18 | Deadline (`other:deadline`) | 7% | 4% | 11% | 4% | [rust RFC #2394](https://github.com/rust-lang/rfcs/pull/2394), [excalidraw #2945](https://github.com/excalidraw/excalidraw/issues/2945) | Rarely | An optional "decide by" date, at most. | Low | +| 19 | Vote (P7; `vote`) | 9% | 4% | 14% | — | [MoJ ADR019](https://github.com/ministryofjustice/modernisation-platform/blob/main/architecture-decision-record/0019-use-bash-go-as-core-languages.md), [tc39 `.at`](https://github.com/tc39/notes/blob/main/meetings/2020-11/nov-17.md#item-rename--revisit-inclusion-on-string) | No | Out of scope | High | +| — | Hidden history (`other:hidden-history`) | 12% | 20% | 3% | 0% | — | — | **Not a design gap.** It's an artefact of webfetch summaries. At most, a record should flag a partial history. | High | + +## 4. What to do + +### (a) Make now: relevant to the persona, and the evidence is strong + +1. **Follow-ups on Settle** (#1): a consequences or obligations field, plus child questions linked to the parent, and a "settled in principle" parent whose children are still open. This is the single biggest gap, it holds on git-file, and it resolves "mixed". +2. **A fate for each option in choice** (#2): chosen / partly / later / not this, each with an optional reason. This replaces single-select and absorbs rank, partition and the reasons for rejected options. +3. **A Parked state with a revisit trigger** (#3, #7): it doesn't block Build plan. "Close without deciding", with a reason, replaces deleting a moot question. One "Revisit if…" field serves both Parked and provisional Settle. Split reversibility out of confidence. +4. **Research and Spike on every shape** (#4): per option on choice, and claims attachable to cells. This revisits hard call G for evidence actions. +5. **Reopen keeps the old answer as superseded** (#8): a forward link from the old answer, and revert PRs in Built into. This is nearly free, since the timeline is already kept. +6. **A "Driven by" field** (#6), for authorities outside Chopin. +7. **Record a decision after the fact** (#5): from a Chat message or PR, with a "Decided in" source. This is also the cheapest way into G3, agent-made calls, once build logs can be tested. + +Keep G1 (Accept Chopin's proposal) from the theory pass. The corpus neither supports nor refutes it. + +### (b) Design for, but defer: these matter for teams + +- **Owner and decider roles**, with hand-over (#11). An approver comes later. +- **Team settle** and an **unanswered-objection warning** on Settle (#12). +- **Depends on / affects links** beyond parent and child (#9). +- **A user-signal evidence type** (#10). Confirm it on full-fidelity data first. +- **Reframe history** and Chopin-proposed reshape (#13). +- **Scope: repo**, exported to agent instructions (#14). Promote it to (a) if the local dataset shows ≥15% of decisions are rules in small repos. +- **Image options** (#15). +- **An optional "decide by" date** (#18). + +### (c) Out of scope: committee and standards-body mechanics + +| Mechanic | Data | Why Chopin shouldn't model it | +| ------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | +| Votes, tallies, ranked ballots | 9% of records, 71% of them from formal collectors. Polls outside standards bodies: 8%. | Not how a solo builder or small team decides. Thumbs already cover a temperature check. | +| Unanimity, any-delegate blocks, explicit-support procedures | `consensus-mechanics` hits are 82% formal, and the tag is in 0/17 of the High stratum. | Governance rules belong to the organisation, not to a planning document. (b) covers the small-team core. | +| Stage gates, FCP timers, review-board columns (Go's Likely Accept and similar) | Formal collectors only. Escalation present in 6/82. | Process machinery. A decision for a solo builder has one gate: Settle. | +| Moderation, thread locking, bots as participants | 3 records (`other`) | Belongs to the platform, not the decision. | +| Release-train and edition deadlines | `deadline` 83% formal | Committee scheduling. At most an optional date (b). | +| Stakeholder-class consent (distributions, CI vendors, an institution's trust) | `crowd-signal` in Go/Python threads | No one is asked. Represent it as evidence, not as a participant. | + +## 5. Shapes + +| Measure | All | Persona | Git-file | +| --------------------------------------- | ------------------------------------------------------------------ | ------------------- | --------------------- | +| choice / explore / mixed / check / tune | 67 / 11 / 11 / 6 / 5% | 71 / 9 / 4 / 7 / 9% | 75 / 10 / 10 / 4 / 0% | +| Has a secondary shape | 67% | 58% | 60% | +| Check as primary or secondary | 32% | 27% | 35% | +| Commonest pairs | choice alone 26; choice+check 19; explore↔choice 15; choice↔tune 8 | | | + +- **Keep four shapes** (moderate confidence). + - The corpus offers no candidate fifth shape: rank and partition are 4%. + - Go/no-go decisions (existence and ban, 40% of records) were all coded as choice. Only 17% of those have ≤3 options, so a compact Yes / No / Not now control isn't evidenced as necessary. It's still plausible for the persona. +- **"Mixed" needs structure, not a shape.** Each of the 9 mixed records is a headline decision with attached sub-choices: [slog](https://github.com/golang/go/discussions/54763), [RFC 3392](https://github.com/rust-lang/rfcs/pull/3392), [RFC 2394](https://github.com/rust-lang/rfcs/pull/2394). Follow-ups (a1) make the parent one shape and each child its own shape. +- **Choice is probably over-dominant because of how it was coded** (moderate confidence). Four things point that way: + - The admission rule requires ≥2 options weighed. + - The schema forces an `options` array. + - Both recoding disagreements went to choice (tune→choice, explore→choice). + - Chance agreement is 0.60, so κ = 0.618 rests on 11/13 raw agreement. + + The correct reading is that "a decision with named alternatives is the commonest _public_ decision". For a solo builder with an agent, explore and accept-a-proposal may well be commoner, and this corpus can't say. +- **Check is better seen as evidence.** Choice+check is the second-commonest pair. This backs a4: claims attach to choice rows, and the standalone check card stays for real verify-a-belief questions (contentKind `verify-belief`: 1 record). +- **The κ caveat:** shape-level percentages carry an uncertainty of roughly ±10 points beyond sampling error. Don't make design decisions on gaps of less than 15 points between shapes. + +## 6. Over-built, according to the corpus + +| Part | Evidence | Verdict | Conf. | +| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | -------- | +| Tune's bespoke live preview for every tune question | Tune 5% overall, 0% on git-file, 24% in ui-ux and product. `visual-comparison` gaps ask for images _on options_, not live dials. | Make the preview optional (dials plus the resulting sentence). Images on choice rows earn more. | Moderate | +| Tally or vote UI | Votes or polls 8.5%. Outside formal collectors 5%. | Don't build it. | High | +| Escalation and review states | Escalation present in 6/82. | Don't build them. | High | +| Criteria matrix as the default for choice | Untestable, because `criteria` is saturated. 3 gaps say pros and cons lived as prose bullets, or that a ✓/½/✕ grid would suggest more rigour than the decision had. | Allow prose cells and optional criteria. Weak evidence. | Low | +| Confidence as a headline fact | Untestable: coding is inconsistent. Reversibility is stated in 34%. | Lead with reversibility and the revisit trigger (a3). | Low | +| **Not over-built:** Takes | Dissent survives in 76% of decided records with ≥3 participants. F4 (ask a teammate) fits 44% of Persona records. | Keep Takes on the record. The _single-face_ "Settled by" is **under**-built for teams: 66% are decided by a team or committee. | Moderate | + +## 7. What the corpus can't tell us, and what to measure next + +| Question | Why this corpus can't answer it | Measure on the local full-fidelity GitHub set (decisions-v1) | Measure in user testing or Chopin logs | +| ---------------------------------------------------- | ------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| G1: is ratifying the agent's proposal the modal act? | Undebated decisions are excluded. Option order isn't chronological. | P(the first-proposed option wins), with order taken from timestamps and split by proposer role (opener / maintainer / contributor). | Share of settled questions where Chopin's lean was accepted unchanged, and the time to Settle, with and without one-click Accept. | +| G3: how many calls do agents make silently? | No PR-review debates (0/82 match H11). | Of the PR-review quota (20): share where the approach changed in review, and share where the debate happened on an implementation PR. | Unasked decisions per agent PR, and the share the builder reverses once they're surfaced. | +| How does a solo builder or a small team behave? | Only 3 records have ≤3 participants. | ≥30 records from repos with fewer than 5 contributors. Recompute every gap rate, and report the difference from the committee stratum. | — | +| How often are questions parked, and for how long? | Stale threads are mostly webfetch product-UI. | Share closed by a stale bot or without a verdict; Hold or postponed labels; median time to revisit. | Parked count per plan; share later revived; share of parked questions that blocked Build plan in the current design. | +| Composite answers | Option lists are inflated by public debate. | Share of decided threads whose final decision combines ≥2 options, with options counted only if someone actually advocated them. | How often people use partly or later fates once they're available. | +| Crowd signal: real or an artefact? | 95% of hits are webfetch. | Correlation between `reactionsTotal` or upvotes and the outcome; share of decisions where the rationale cites reaction counts. | — | +| Shape reliability | κ 0.618 on 13 records. | Double-code ≥20% of records, allowing "fits two shapes". Target κ ≥0.7, and report κ excluding choice. | Share of questions reshaped (F8) after creation. | +| Conventions for agents | n = 12; the scope of a rule is rarely stated. | Share of decisions later reflected in CONTRIBUTING, AGENTS.md or lint config. | Share of settled questions a builder marks as repo-wide; whether agents violate them later. | +| Follow-ups in use | Tag priming. | Double-code `nested`, `staged` and `consequences` on a 15% sample (κ). | Follow-ups created per Settle; share of child questions settled before Build plan. | +| Confidence against reversibility | Inconsistent coding. | Re-code `confidence` as "a level was expressed" (yes / no). | Which fact readers use when deciding whether to Reopen. | diff --git a/evals/datasets/decisions-v1/analysis/critique-coverage-theory.md b/evals/datasets/decisions-v1/analysis/critique-coverage-theory.md new file mode 100644 index 000000000..885afb584 --- /dev/null +++ b/evals/datasets/decisions-v1/analysis/critique-coverage-theory.md @@ -0,0 +1,204 @@ +> **Provenance.** Written in an earlier cloud session against corpus v0 (82 records, including webfetch summaries). Paths such as `review/…`, `decisions/…` and `SPEC.md` refer to that session's scratchpad. Kept unchanged as the basis for [coverage-critique.md](coverage-critique.md), which reruns the tests on decisions-v1. + +# Critique: coverage. Does the questions design support the decisions builders actually make? + +Scope: the design as agreed in review/SPEC.md, review/consult.md (§1 layers, §8 flows F1–F8, §9 hard calls), review/critique-pages.md (unified page) and canvas/project/TuneExamples.dc.html. This is a theory-first pass: decisions/raw/ is still empty, so every frequency claim below is a prediction to be tested against decisions/SCHEMA.md records, not a finding. + +**Confidence key.** _High_: established, I would stake the recommendation on it. _Moderate_: well-grounded reasoning or a source I am fairly sure of, but I have not checked the exact wording. _Low_: informed guess; the dataset should decide it. + +**One structural caveat before anything else (high).** The corpus is collected from public threads (RFCs, proposals, issues, meeting notes) and only admits decisions that were _debated_ (SCHEMA "Only decisions that were actually DEBATED…"). That oversamples committees, consensus and long threads, and it structurally _excludes_ the decisions this persona makes most: undebated calls between one person and an agent, and calls an agent makes silently in code. Hypotheses below are therefore phrased with a stratum (for example `participants.count ≤ 3`, or a non-API `domain`) where the stratum matters. Two gaps (G3, G8) cannot be fully tested on this corpus at all; I say so where it applies. + +--- + +## 1. Sources I lean on, and how sure I am of what they say + +| Source | What I use from it | Confidence | +| ---------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | +| Kruchten (2004), "An ontology of architectural design decisions in software-intensive systems" | Three decision kinds: **existence** (including the negative "ban" decisions), **property** (constraints, rules, qualities), **executive** (process, technology, tools, organisation). States roughly _idea, tentative, decided, approved, challenged, rejected, obsolesced_. Relations such as _constrains, forbids, enables, subsumes, conflicts with, overrides, comprises, is an alternative to, is related to_. | Kinds: high. State list and relation list: moderate (I am sure of the idea, less so of every label). | +| Nygard (2011), "Documenting Architecture Decisions" (blog post; the origin of ADRs) | Record = Title, Context, Decision, **Status** (proposed / accepted / deprecated / **superseded**), **Consequences**. ADRs are append-only: you supersede, you do not edit. | High | +| Tyree and Akerman (2005), "Architecture decisions: demystifying architecture", IEEE Software | A richer template with **assumptions**, constraints, positions, argument, implications, **related decisions**. | Moderate | +| MADR template | "Considered options", "decision drivers", pros and cons per option. | Moderate | +| QOC: MacLean et al. (1991), "Questions, Options, and Criteria" | The design-rationale ancestor of the choice shape (options × criteria). | High | +| IBIS (Kunz and Rittel, 1970) and gIBIS (Conklin and Begeman, 1988) | Issue → positions → arguments, and issues that _spawn_ sub-issues. The ancestor of explore + Arguments, and of nested questions. | High | +| Rust RFC process | RFCs carry an explicit **"Unresolved questions"** section; acceptance goes through a **Final Comment Period** whose disposition is merge, close or **postpone**; a team member can file a blocking concern. Acceptance ≠ stabilisation: details are settled later on the tracking issue. | High for unresolved questions, FCP and postpone; moderate for the concern mechanics. | +| Python PEP 1 | Statuses include **Deferred, Provisional, Withdrawn, Rejected, Superseded**, Accepted, Final. Decided by the Steering Council or a PEP-Delegate. | High | +| Go proposal process | A review group moves issues to "likely accept/decline", then a final comment period, then Accepted / Declined / **Hold**. | Moderate | +| TC39 | Staged maturity (stages 0–4, with an intermediate stage added recently); consensus of delegates, where one objection can block advancement. | High for stages and consensus; moderate for the newer intermediate stage. | +| Apache voting | +1 / 0 / −1, **lazy consensus**, a justified −1 on a code change acts as a veto. | High | +| IETF RFC 7282 (Resnick, 2014), "On Consensus and Humming in the IETF" | Rough consensus means objections are _addressed_, not out-voted. | High | +| Vroom and Yetton (1973) | Decision styles from autocratic through consultative to group (AI, AII, CI, CII, GII). | High | +| Rogers and Blenko (2006), "Who has the D?", HBR (Bain's RAPID) | Recommend, **Agree** (a veto-holding sign-off), Perform, Input, **Decide**. DACI (Driver, Approver, Contributors, Informed) is the common product-team variant. | High for RAPID; moderate for where DACI came from, so I don't name an origin. | +| Amazon shareholder letters (around 2015–16) | Type 1 (one-way door) versus Type 2 (two-way door) decisions; "disagree and commit". | Moderate on the year, high on the content. | +| Poppendieck and Poppendieck (2003), _Lean Software Development_ | **Last responsible moment**: deferring is a deliberate, legitimate decision. | High | +| Hammond, Keeney and Raiffa (1999), _Smart Choices_ (PrOACT) | Problem, Objectives, Alternatives, Consequences, Trade-offs, plus uncertainty, risk tolerance and **linked decisions**. | Moderate-high | +| Klein (1998), _Sources of Power_ (recognition-primed decision); Simon on satisficing | Experienced people usually evaluate **one** candidate and adapt it, rather than comparing many on criteria. | High | +| Shape Up (Singer, 2019) | Appetite, the betting table, scope hammering: sequencing and scope decisions as a recurring cycle. | High | +| GitHub Spec Kit (2025) | Agents mark ambiguities in a spec as "needs clarification" and resolve them in a clarify step: prior art for agent-found questions. | Moderate | + +I have deliberately left out empirical survey numbers (for example from the design-rationale surveys), because I can't recall their figures reliably. + +--- + +## 2. Taxonomy and coverage map + +Legend: **COVERED** (works as designed), **PARTLY** (doable, but awkward, lossy or a workaround), **MISSING** (no way to express it). "Persona frequency" is my estimate for a solo builder working with agents: H, M or L (low confidence unless stated). + +### 2a. Content kind: what the decision is about + +| # | Kind | Example in a Chopin plan | Literature | Coverage | How, or why not | Persona freq. | +| --- | -------------------------------------------------------------------------- | ---------------------------------------------------- | ----------------------------------------------------- | ---------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------- | +| K1 | **Selection** among named alternatives (library, approach, provider) | "SQLite, Postgres or KV for drafts?" | QOC; MADR; Kruchten executive (technology) | **COVERED** | Choice table. | H | +| K2 | **Existence / go–no-go** (do we build X at all) | "Do we need offline mode in v1?" | Kruchten existence | **PARTLY** | Explore (free text) or a two-row choice. There's no compact Yes / No / Not now; "Not now" has no state (see L3). | H | +| K3 | **Ban / non-goal** (we will _not_ do X) | "No user accounts in v1." | Kruchten ban (anti-existence) | **PARTLY** | Settles as prose, but nothing marks it as a constraint that build agents must respect. Agents build what is not forbidden. | M | +| K4 | **Property / quality constraint** | "p95 save under 200 ms", "WCAG AA" | Kruchten property | **PARTLY** | Explore or tune. No notion of a threshold that becomes an acceptance check for the PRs in "Built into". | M | +| K5 | **Convention / standing rule** (applies everywhere, not just here) | "All errors go through `Result`; never throw." | Kruchten property (rules, guidelines) | **PARTLY** | Settles inside one plan. There's no _scope_ (this plan / this repo) and no way out to agent instructions. | M–H (moderate) | +| K6 | **Executive: process, tooling, people** | "GitHub Actions or none?", "Who reviews agent PRs?" | Kruchten executive; RAPID | **PARTLY** | Tool choice = choice (covered). People and process (who, cadence) mostly out of scope for a solo builder. | L–M | +| K7 | **Naming** | "`plan`, `draft` or `doc` for the object?" | (TC39 naming threads are a common corpus case) | **PARTLY** | Choice with criteria is heavy for naming; explore loses the candidate list. What's needed is candidates + quick takes, criteria optional. | M | +| K8 | **Default / parameter value** | "Retry: 5 attempts, exponential, jitter." | (TuneExamples) | **COVERED** where there's a visible effect; **PARTLY** otherwise | Tune with a preview. A bare number (a timeout) gains little from a bespoke preview. | M | +| K9 | **Perceptual trade-off tuning** (density, tone, wording) | "How dense is the run table?" | — | **COVERED** | Tune is strong here, and is the design's most original idea. | M | +| K10 | **Verification of a belief** | "Is it true most people leave before writing?" | (spikes; hypothesis-driven development) | **COVERED** | Check + Research / Spike. | M | +| K11 | **Assumption** (unverified, proceed anyway) | "Assume < 1k plans per user until we see otherwise." | Tyree and Akerman (assumptions); Kruchten _tentative_ | **PARTLY** | Check forces True / Partly / False. "Unknown, assume X" is the commonest real outcome of a check, and it has no verdict. The Shapes board had "Unverified"; the spec dropped it. | H (moderate) | +| K12 | **Interface / data-model design** (a proposed artefact to accept or amend) | "Here's the `Plan` schema; OK?" | IBIS position; ADR | **PARTLY** | Explore holds prose, not a schema or code sketch; no "accept / amend this proposal" outcome. | H | +| K13 | **Sequencing / priority** | "Which of these four first?" | Shape Up betting | **MISSING** | Choice picks one row; nothing ranks. | M | +| K14 | **Scope partition** (which of N go in v1) | "Of 9 features, which are v1?" | MoSCoW; Shape Up scope hammering | **MISSING** | Choice is single-select; there's no multi-select or in/out. | M | +| K15 | **Risk acceptance / exception** | "Ship without rate limiting for the beta." | — | **PARTLY** | Explore. Needs an expiry or a revisit trigger (see L4). | L–M | +| K16 | **Reversal** of an earlier decision | "We said SQLite; move to Postgres." | Nygard superseded; PEP Superseded | **PARTLY** | Reopen edits in place; there's no "superseded by", and nothing about code that already shipped. | L–M | + +### 2b. Structure: how decisions relate + +| # | Structure | Example | Literature | Coverage | How, or why not | Persona freq. | +| -- | --------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------- | ----------- | ---------------------------------------------------------------------------------------------------------------------- | -------------- | +| S1 | **Single, independent** | "Warm or neutral tone?" | — | **COVERED** | Every flow assumes this. | H | +| S2 | **Coupled** (jointly constrained: settle together or stay consistent) | "Where progress lives" ↔ "how reset works" (the design's own example: Sam's objection on Q2 is really a second question) | Kruchten _constrains / conflicts with_; PrOACT _linked decisions_; design structure matrices | **MISSING** | No relation between questions. Workaround: a choice whose rows are combinations (lossy, and explodes combinatorially). | M–H (moderate) | +| S3 | **Blocking / precedence** (B can't be answered until A is) | "Pricing tiers" waits on "do we have accounts?" | Kruchten _enables_; RFC prerequisites | **MISSING** | The capsule counts open questions flat; nothing says which to do first. | M | +| S4 | **Nested / decomposed** (a parent spawns sub-questions) | "Offline mode?" → storage, conflict policy, sync badge | IBIS sub-issues; Kruchten _comprises_; corpus field `subQuestions` | **MISSING** | The child page has Findings / Spike sub-pages but not sub-_questions_. | M | +| S5 | **Staged** (accept in principle, details later) | "Yes to offline; conflict policy TBD." | Rust RFC "unresolved questions" → stabilisation; TC39 stages; PEP Provisional | **MISSING** | Settle is all-or-nothing. | M | +| S6 | **Conditional** (outcome depends on a future fact) | "Use KV unless the spike shows > 50 ms reads." | — | **MISSING** | Can be written into an explore answer, but nothing watches the condition. | L–M | +| S7 | **Recurring / precedent** (the same class of call again) | "Which major dependency bumps to take?", "Do we add a flag for this?" | Zimmermann's reusable decision models (moderate) | **MISSING** | No lookup of "we decided this before". The Decisions view is itself marked missing (F7.3). | M | +| S8 | **Batch of small independent calls** | Chopin finds 9 micro-questions in one plan | — | **PARTLY** | One expanded card at a time; no triage pass, no "accept all of Chopin's leans". | H (moderate) | + +### 2c. Social mode: who decides + +| # | Mode | Example | Literature | Coverage | How, or why not | Persona freq. | +| --- | -------------------------------------------------------------------------------- | ---------------------------------------------------------------- | --------------------------------------------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | +| P1 | **Solo with agent** | Ada and Chopin | Vroom–Yetton AI | **COVERED** | The core loop. | H | +| P2 | **Consult** (ask, then decide alone) | Ada asks Luke for a take | Vroom–Yetton CI/CII; RAPID Input | **COVERED** | Ask popover, takes, waiting ring. | M | +| P3 | **Agent decides, human ratifies later** | Build agent picks a debounce of 300 ms; Ada confirms or reverses | (no established framework; new to agent work) | **MISSING** | "Settled by" is always a person; there's no "Chopin settled it, unconfirmed" state. | H (moderate) | +| P4 | **Delegate** (someone else owns the call) | "Luke, your call on the icon set." | RAPID Decide; DACI Driver | **MISSING** | Askees can't Settle (L6 withholds it). | L–M | +| P5 | **Approval / sign-off** (decider proposes, an approver must agree) | Security or a client must OK auth flow | RAPID Agree; DACI Approver; Apache −1 veto | **MISSING** | Takes are advisory; "Objects" doesn't block Settle. | L (solo), M (small team) | +| P6 | **Consensus / consent** (everyone can live with it) | Small team agrees an API shape | IETF rough consensus; TC39 | **PARTLY** | Agrees / Objects / Unsure are visible, but there's no "all asked have responded / no open objection" signal. "Can live with it" (the consent stance) was retired from the vocabulary. | L | +| P7 | **Vote** | Pick a name by count | Apache +1/0/−1 | **PARTLY** | Thumbs exist; no tally. Rare for this persona; fine. | L | +| P8 | **External authority** (client, platform rule, regulator, manager not in Chopin) | "App Store rules force IAP." | — | **PARTLY** | Recordable in explanation or "What we knew", but "Settled by" names the clicker, not the actual decider. | L–M | +| P9 | **Committee / formal process** (FCP, stages) | Rust FCP | Rust, TC39, Go | **MISSING, out of scope** | Correctly out of scope for the persona; expect the corpus to over-represent it. | L | +| P10 | **Crowd signal** (users, telemetry, issue reactions) | "Most-thumbed issue wins" | — | **PARTLY** | Research on a check can pull it in; there's no data-source evidence type. | L | + +### 2d. Evidence mode: what the decision rests on + +| # | Mode | Example | Coverage | How, or why not | Persona freq. | +| -- | --------------------------------------------------------------- | --------------------------------------- | ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------- | +| E1 | **Judgement / opinion** | "Warm feels right." | **COVERED** | Explore, Takes, Arguments. | H | +| E2 | **Desk research / data** (docs, benchmarks, analytics) | "Does lib A support streaming?" | **PARTLY** | Research exists **on check only** (hard call G1). A choice cell like "lib A: streaming ✓?" is itself a check, but can't be researched from the table. | H | +| E3 | **Prototype / spike** | "Build both and compare." | **PARTLY** | Spike exists on check only. Agents make multi-variant spikes cheap, which is the agent era's biggest change to how choices get made; choice has no "spike each option". Tune's preview is a lightweight prototype (covered). | M–H | +| E4 | **Experiment in production** (flag, A/B, measure) | "Ship both onboarding variants to 10%." | **MISSING** | Needs a provisional settle plus a metric and a date (S5/S6/L4). | L | +| E5 | **External constraint** (licence, platform, compatibility, law) | "MIT-only dependencies." | **PARTLY** | Lives as text in "What we knew"; nothing marks it as forcing the outcome. | M | +| E6 | **Precedent** (earlier decision, convention) | "Same as we did for exports." | **MISSING** | No link to other questions or records (S7). | M | + +### 2e. Lifecycle: the states a decision passes through + +| # | State | Literature | Coverage | How, or why not | Persona freq. | +| --- | --------------------------------------------------------------- | ------------------------------------------------------------- | ---------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------- | +| L1 | Open / proposed | Nygard proposed; Kruchten idea | **COVERED** | Open pill, neutral disc. | H | +| L2 | Waiting on a person / researching / spiking | — | **COVERED** | Waiting for Luke, Researching, Building a spike. | M | +| L3 | **Deferred / parked / postponed** ("not now") | PEP Deferred; Rust postpone; Go Hold; last responsible moment | **MISSING** | The only exits are Settle and Discard. A parked question keeps the capsule amber and blocks "Build plan". | H (moderate-high) | +| L4 | **Provisional / tentative** ("go with X for now; revisit if Y") | Kruchten tentative; PEP Provisional; Type 2 doors | **PARTLY** | Confidence "Medium, easy to change" gestures at it. There's no revisit trigger: the board 7 line "If it is often, this is worth reopening" was cut as reader-directed copy, but it was actually a _field_. | H (moderate) | +| L5 | **Accepted in principle, details pending** | Rust RFC → stabilisation; TC39 stages | **MISSING** | Needs S4/S5. | M | +| L6 | Settled / accepted | Nygard accepted | **COVERED** | Margin marker, record. | H | +| L7 | Challenged / reopened | Kruchten challenged | **COVERED** | Reopen (F6), with the timeline kept. | M | +| L8 | **Superseded** (replaced by a newer decision) | Nygard; PEP Superseded | **PARTLY** | Reopen rewrites the same record; there's no forward/back link between old and new answers once code shipped on the old one. | L–M | +| L9 | **Reverted after shipping** | — | **MISSING** | "Built into" lists PRs; there's no revert PR, and the record doesn't learn that the code was undone. | L | +| L10 | **Dissolved / withdrawn / moot** (question no longer matters) | PEP Withdrawn; Kruchten rejected | **PARTLY** | Discard deletes the question _and its record_; the reason it stopped mattering is lost. | M | +| L11 | **Obsolesced** (the feature it governed was removed) | Kruchten obsolesced; Nygard deprecated | **MISSING, low value** | Probably out of scope. | L | +| L12 | **Implicit / decided in code** (never asked; found later) | (design-rationale literature: much rationale goes unrecorded) | **MISSING** | F1 only finds questions at planning time. | H for agent builders (moderate) | + +### 2f. Record fields against decision-record practice + +| Field (practice) | In the design? | Note | +| ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | +| Decision, context (Nygard) | **COVERED**: answer, explanation, "What we knew" | — | +| Options considered (MADR) | **COVERED** for choice; **PARTLY** for others via Arguments (if critique-pages §4 is adopted) | — | +| Status incl. superseded (Nygard) | **PARTLY**: Settled / Open only | See L3, L4, L8. | +| **Consequences / follow-ups** (Nygard; Tyree and Akerman "implications") | **MISSING** | What becomes easier or harder, and which new questions this spawned. For agents this is the most build-relevant part. | +| **Assumptions** (Tyree and Akerman) | **PARTLY**: "Not known" rows | Unknowns are listed; assumptions _made despite_ the unknowns are not. | +| **Related decisions** (Tyree and Akerman; Kruchten relations) | **MISSING** | S2–S5. | +| Dissent | **PARTLY** | Only if Takes survive onto the record (critique-pages problem 2). | +| Reversibility (Type 1/2) | **PARTLY**: folded into the confidence string | consult §2 (7-24) recommended splitting it; the spec kept one value. | +| **Revisit trigger / review date** | **MISSING** | Cut as copy; should be a field. | +| Scope (where it applies) | **MISSING** | K5. | +| Implemented in | **COVERED**: Built into | Revert not modelled (L9). | + +--- + +## 3. Top gaps, ranked by how often a solo builder working with agents would hit them + +Ranking confidence overall: **moderate** for the top four as a group, **low** for the exact order below them. + +| Rank | Gap | Concrete decision | What it needs | Inside an existing shape? | Confidence (gap is real / rank) | Testable prediction | +| ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **G1** | **Ratify the agent's recommendation in one act.** The modal real decision is one candidate, accepted or tweaked (Klein; satisficing). The spec makes untouched defaults non-answers (SPEC §6, consult F2.4), so accepting Chopin's lean takes a fake edit. Choice requires ≥2 rows; explore has no "Chopin's proposal" to accept. | "Chopin proposes `drafts` table with `(id, plan_id, body, updated_at)`. Fine." | New **field**: answer origin (proposed by Chopin / chosen by a person) and an explicit **Accept** act (one click, logged as such). No new shape. | Yes, all four shapes: explore pre-filled with Chopin's draft, choice with its lean, tune with its default, check with its verdict. | High / moderate | In decided records with ≥2 options, the option proposed **first** (the champion's) is chosen in **≥60%** (`options[0].fate == chosen`, provided collectors list options in proposal order; check `notes`). | +| **G2** | **Defer ("not now").** Solo builders park questions all the time; the last responsible moment is a legitimate outcome, and every mature process has the state (PEP Deferred, Rust postpone, Go Hold). Today a parked question blocks "Build plan" or gets Discarded. | "Team plans? Not for v1; ask again when we hit 50 users." | New **state** Deferred (neutral, doesn't count in the capsule, doesn't block Build plan) + optional **revisit trigger** field. | Yes: a state, not a shape. | High / moderate-high | `process.deferred == true` in **≥15%** of all records; among `outcome.decided == false`, deferred **≥** abandoned (`how == abandoned`). | +| **G3** | **Decisions made by agents during build** that nobody asked: surfaced afterwards, to ratify or reverse. F1 only finds questions while planning; agents then make dozens of unasked calls in code. | Build agent chose optimistic updates for the save button; Ada should see it as a settled-by-Chopin question. | New **origin** ("found in build", linked to the PR) + new **state** "Settled by Chopin, unconfirmed" (P3). No new shape; reuses the record. | Yes. | Moderate / moderate. The corpus barely sees it (undebated calls are excluded). | **≥20%** of records have a `pr` in `sources[].kind` **and** `process.spike == implementation-pr`, i.e. the debate happened on code that already embodied a decision. (Weak proxy; confirm on Chopin build logs.) | +| **G4** | **Coupled and blocking questions.** Answers constrain each other (storage ↔ reset is the design's own running example) or must come in order. No relation exists, so Ada settles A, then finds B silently invalid. | "Store progress on the account" forces "reset needs a support tool". | New **relation** between questions: _depends on_ (blocks) and _constrains_; settling A flags B "affected". Capsule hover can order by dependency. | Yes for the questions; needs a relation layer. | Moderate / moderate | **≥20%** of records have non-empty `process.subQuestions` **or** a `chopinFit.gaps` entry mentioning coupling, dependency or ordering. | +| **G5** | **Provisional settle with a revisit trigger** ("for now"). Distinct from G2: build proceeds on it. Kruchten tentative, PEP Provisional, Type 2 doors. The cut "worth reopening if…" line was this field. | "KV for now; revisit if reads exceed 50 ms or we add search." | New **field** "Revisit if…" on Settle, optional; the marker can carry a subtle provisional variant. Confidence and reversibility split (consult 7-24). | Yes. | Moderate-high / moderate | `outcome.reversibility ∈ {stated-easy, stated-hard}` in **≥25%** of decided records, while `outcome.confidence == stated` in **≤15%** (reversibility is discussed more often than confidence). | +| **G6** | **Evidence on every shape, not just check.** Research and Spike live on check only (hard call G1). Choice cells _are_ claims ("lib A streams ✓?"), and agents can spike each option cheaply. | "Spike both the CRDT and last-write-wins versions and show me the conflicts." | Allow Research / Spike on choice (per option or per cell) and explore; no new shape. Revisits hard call G toward G2 for evidence actions only. | Yes. | Moderate / moderate | Among records with `chopinFit.shape ≠ check`, **≥30%** have `process.research ≠ none` or `process.spike ≠ none`. | +| **G7** | **Nested / staged: accept in principle, details later.** The Rust RFC "unresolved questions" pattern. Today settling the parent either lies (details aren't settled) or waits (blocks everything). | "Yes to offline mode" settled; "conflict policy" and "sync badge" spawned as child questions. | New **relation** parent → child questions + **state** "Settled in principle" (parent settled, children open). Child questions appear in the plan under the parent's sentence. | Yes; relation + state. | Moderate / moderate-low | **≥15%** of all records, and **≥40%** of those whose `sources[].kind` includes `rfc` or `proposal`, have non-empty `process.subQuestions` with `outcome.decided == true`. | +| **G8** | **Scope: conventions that outlive the plan.** Many solo-plus-agent decisions are standing rules ("always zod at boundaries"), which must reach every future agent run, not just this plan. | "No new runtime dependencies without asking." | New **field** scope: this plan / this repo; repo-scoped records exported to agent instructions. No new shape. | Yes. | Moderate / low | **≥15%** of records are rules or conventions (coder tag in `notes`, or `domain ∈ {process-governance, tooling}` with a question phrased as "should we always / never"). Partly untestable on this corpus. | +| G9 | Sequencing and scope partition (rank; in/out of N) | "Which 3 of 9 features are v1?" | **New shape** (a rank/partition list) _or_ choice with multi-select + order. I'd try multi-select choice first. | Partly (choice multi-select). | Moderate / low | **≥10%** of records are sequencing or scope decisions (options with fates `chosen` and `deferred` mixed in one record). | +| G10 | Supersede and revert after shipping | "We shipped SQLite; now Postgres. PR #212 reverted #188." | New **relation** supersedes / superseded-by; new state Reverted; "Built into" shows revert PRs. | Yes. | High / low frequency | `process.reopened` or `process.superseded` true in **≥10%**; of those with non-empty `implementedIn`, **≥30%** have a gap mentioning revert or migration. | +| G11 | Dissolved questions keep a record | "Moot: we dropped the feature." | Discard splits into **Withdraw (keeps a one-line record)** and Delete. | Yes. | Moderate / low | `outcome.how == abandoned` in **≥10%** of records. | +| G12 | Delegate and approve | "Luke decides icons"; "security must sign off auth." | New **role** on a question: decider ≠ owner; optional approver whose Objects blocks Settle (RAPID Agree). | Yes. | High that it's real / low for this persona | Among records with `participants.count ≥ 3`, `participants.decider` is someone other than the proposer in **≥40%**; `process.escalation ≠ none` in **≥10%**. | + +**What I would not add.** Formal votes, quorum, FCP timers and stage gates (P7, P9). The corpus will make them look frequent because of where it samples; for this persona they are out of scope (confidence: high). + +--- + +## 4. What is over-built relative to how real decisions go + +| Part | Why it's likely over-built | Confidence | Prediction that would prove me wrong | +| -------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ | ------------------------------------------------------------------------------------------------------------------- | +| **Choice's editable criteria matrix** (✓ / ½ / ✕ cells, add-criterion) as the default for choice | Recognition-primed decisions and satisficing (Klein; Simon): most real choices weigh 2 options informally, and criteria are rarely made explicit. QOC-style matrices are the minority. Keep the table, but make criteria optional and allow a two-option "A or B" with Arguments only. | Moderate | `criteria` non-empty in **> 60%** of choice-shaped records, or median `options.length ≥ 4`. | +| **Tune's bespoke live preview for every tune question** | Each of the five TuneExamples needs custom rendering (table, sentence, calendar strip, time axis, result list). A great idea for perceptual calls, but expensive and rare. Allow tune without a preview (dials + a resulting sentence). | Moderate | `chopinFit.shape == tune` in **> 15%** of records. | +| **Confidence as a headline fact** | Real threads rarely state confidence; they do argue about reversibility and cost. Lead with reversibility and a revisit trigger, and demote confidence. | Moderate | `outcome.confidence == stated` in **> 30%**. | +| **The ask-a-teammate social layer's richness** (thumbs, thumbs-down, not sure, emoji, reply, withdraw with a struck row, multi-askee waiting copy) | It's polished beyond its frequency for a solo builder, while delegation, approval and "everyone has responded" (the social modes that _change who decides_) are absent. Not wrong; mis-weighted. | Moderate-low | In the `participants.count ≤ 3` stratum, `outcome.how == consensus` or `process.poll == true` in **> 30%**. | +| **A full generated record for every question** (timeline, What we knew, With, Confidence) | Most micro-decisions need the sentence and the marker; the record earns its keep for Type 1 doors and contested calls. Generate it lazily (on Open record), or scale it by reversibility and number of participants. | Moderate | Hard to test on this corpus (it only admits debated decisions); test on Chopin usage: share of records ever opened. | +| **Seven disclosure layers** | For most questions L1 → L2 → L4 is the whole journey; L3 and L5 are already converging (critique-pages). Not a coverage issue, but the depth suggests effort spent on the deep end of a distribution that is mostly shallow. | Low | — | +| **Arguments as a separate section from Takes** (if F1 is chosen) | Real threads mix stance and reason; splitting them relies on Chopin extracting arguments reliably (consult F's own caveat). | Low | — | + +What's **right-sized**: check (verification and spikes are real and frequent), the Reopen flow with its retained timeline, "Built into" (traceability to PRs is something ADR practice wishes it had), and the rule that settled questions melt into prose (decisions belong next to what they govern). + +--- + +## 5. Hypotheses for the dataset pass (checklist) + +Each line names the SCHEMA fields that falsify it. "All records" means the whole corpus; strata are given where the corpus bias matters. Before testing H1, H2, H6 and H7 (which rest on coder judgement), **double-code a 15% sample** of `chopinFit.shape` and `chopinFit.gaps`; if Cohen's κ < 0.6 on shape, treat the shape-based results as inconclusive. + +- [ ] **H1. The four shapes miss a real share.** `chopinFit.shape ∈ {mixed, none}` in **≥15%** of all records. _Falsified if < 10%._ +- [ ] **H2. Tune is rare; choice dominates.** `shape == choice` is the most frequent single value, and `shape == tune` is **≤10%** of all records (also report the `domain ∈ {ui-ux, product-behaviour}` stratum, where I expect tune to be higher). _Falsified if tune > 15% overall._ +- [ ] **H3. Choices are small and criteria are mostly implicit.** Among choice-shaped records, median `options.length ≤ 3`, and `criteria` is non-empty in **≤50%**. _Falsified if median ≥ 4 or criteria present in > 70%._ +- [ ] **H4. The first proposal usually wins (ratify is the modal act, G1).** In decided records with ≥2 options, `options[0].fate == chosen` in **≥60%**. _Requires collectors to list options in proposal order; drop records where `notes` says otherwise. Falsified if < 45%._ +- [ ] **H5. Deferral is common enough to need a state (G2).** `process.deferred == true` in **≥15%** of all records, and among `outcome.decided == false`, deferred count **≥** `outcome.how == abandoned` count. _Falsified if deferred < 8%._ +- [ ] **H6. Decisions don't stand alone (G4, G7).** Non-empty `process.subQuestions`, or a `chopinFit.gaps` entry coded _coupled / dependent / staged_, in **≥20%** of all records; for `sources[].kind ∈ {rfc, proposal}`, **≥40%**. _Falsified if < 10% overall._ +- [ ] **H7. Evidence is needed beyond check (G6).** Among records with `chopinFit.shape ≠ check`, **≥30%** have `process.research ≠ none` or `process.spike ≠ none`. _Falsified if < 15%._ +- [ ] **H8. Reversibility beats confidence (G5, over-build).** `outcome.reversibility ∈ {stated-easy, stated-hard}` in **≥25%** of decided records, and `outcome.confidence == stated` in **≤15%**. _Falsified if confidence is stated at least as often as reversibility._ +- [ ] **H9. Dissent survives decisions (keep Takes on the record).** `outcome.dissent` non-null in **≥25%** of decided records with `participants.count ≥ 3`. _Falsified if < 10%._ +- [ ] **H10. Decisions are revisited, and in-place Reopen is not always enough (G10).** `process.reopened || process.superseded` in **≥10%** of all records; of those, **≥30%** carry a gap coded _supersede / revert / migration_. _Falsified if reopened-or-superseded < 5%._ +- [ ] **H11. Many debates happen on code that already decided something (G3 proxy).** `sources[].kind` includes `pr` **and** `process.spike == implementation-pr` in **≥20%** of all records. _Falsified if < 10%. A pass doesn't prove G3 for agents; a fail weakens it._ +- [ ] **H12. Small groups decide by one person, not by vote (social-layer weighting).** In records with `participants.count ≤ 3`, `participants.decider` is a single person (named maintainer, BDFL or equivalent) in **≥75%**, and `process.poll == true` or `outcome.how == vote` in **≤10%** of _all_ non-standards-body records (exclude collector `tc39` and other committee collectors). _Falsified if votes or polls exceed 20% outside standards bodies._ + +--- + +## 6. Notes for whoever codes the corpus + +- Add a controlled vocabulary to `chopinFit.gaps` so that H6, H10 and the gap counts are countable without re-reading. I suggest a prefix tag on each entry: `[coupled]`, `[blocking]`, `[nested]`, `[staged]`, `[conditional]`, `[deferred]`, `[provisional]`, `[supersede]`, `[revert]`, `[ratify]`, `[implicit]`, `[delegate]`, `[approve]`, `[vote]`, `[rank]`, `[partition]`, `[convention]`, `[evidence-not-check]`, `[external-authority]`, `[moot]`. +- Record option order as chronological (needed for H4), or say otherwise in `notes`. +- Tag the Kruchten kind (existence / ban / property / executive) and the content kind K1–K16 in `notes` until the schema has a field; it costs one token and makes §2a testable. +- Report every percentage twice: whole corpus, and the `participants.count ≤ 3` stratum. The second is the closest the corpus gets to this persona. diff --git a/evals/datasets/decisions-v1/analysis/tag-reliability.json b/evals/datasets/decisions-v1/analysis/tag-reliability.json new file mode 100644 index 000000000..d20f14691 --- /dev/null +++ b/evals/datasets/decisions-v1/analysis/tag-reliability.json @@ -0,0 +1,365 @@ +{ + "method": "60 document gaps sampled at random (seed 11), recoded blind from text alone with the brief and a calibration set excluding the sampled gaps; compared with the v0 coder's tags.", + "items": [ + { + "id": "eng-backstage-adr013-014-http-fetch-library", + "i": 1, + "v0": "provisional", + "recoded": "conditional" + }, + { + "id": "eng-backstage-adr013-014-http-fetch-library", + "i": 3, + "v0": "convention", + "recoded": "partition" + }, + { + "id": "eng-cloud-platform-adr003-040-ci-tooling", + "i": 0, + "v0": "supersede", + "recoded": "supersede" + }, + { + "id": "eng-cloud-platform-adr003-040-ci-tooling", + "i": 1, + "v0": "external-authority", + "recoded": "external-authority" + }, + { + "id": "eng-cloud-platform-adr012-021-036-048-multi-cluster", + "i": 3, + "v0": "staged", + "recoded": "staged" + }, + { + "id": "eng-cloud-platform-adr012-021-036-048-multi-cluster", + "i": 5, + "v0": "consensus-mechanics", + "recoded": "delegate" + }, + { + "id": "eng-cloud-platform-adr018-019-ingress-controllers", + "i": 0, + "v0": "revert", + "recoded": "evidence-not-check" + }, + { + "id": "eng-govuk-infrastructure-adr018-concourse-ci", + "i": 3, + "v0": "coupled", + "recoded": "coupled" + }, + { + "id": "eng-govuk-infrastructure-adr019-restructure-cicd", + "i": 2, + "v0": "rejected-content", + "recoded": "rejected-content" + }, + { + "id": "eng-k8s-kep2221-remove-dockershim", + "i": 4, + "v0": "consequences", + "recoded": "consequences" + }, + { + "id": "eng-k8s-kep2579-pod-security-admission", + "i": 0, + "v0": "supersede", + "recoded": "supersede" + }, + { + "id": "eng-modernisation-platform-adr002-003-access-management", + "i": 0, + "v0": "rejected-content", + "recoded": "rejected-content" + }, + { + "id": "eng-modernisation-platform-adr002-003-access-management", + "i": 1, + "v0": "consequences", + "recoded": "rejected-content" + }, + { + "id": "eng-modernisation-platform-adr004-019-core-languages", + "i": 2, + "v0": "convention", + "recoded": "convention" + }, + { + "id": "eng-modernisation-platform-adr032-ncsc-pdns", + "i": 2, + "v0": "rejected-content", + "recoded": "other" + }, + { + "id": "eng-modernisation-platform-adr032-ncsc-pdns", + "i": 3, + "v0": "other", + "recoded": "other" + }, + { + "id": "eng-modernisation-platform-adr043-s3-encryption-scp", + "i": 0, + "v0": "stale-idle", + "recoded": "stale-idle" + }, + { + "id": "eng-nodejs-tsc-2026-ncrypto-source-of-truth", + "i": 2, + "v0": "implicit", + "recoded": "implicit" + }, + { + "id": "eng-nodejs-tsc-2026-ncrypto-source-of-truth", + "i": 4, + "v0": "blocking", + "recoded": "blocking" + }, + { + "id": "python-2007-pep-3103-switch-statement-rejected", + "i": 1, + "v0": "delegate", + "recoded": "delegate" + }, + { + "id": "python-2007-pep-3103-switch-statement-rejected", + "i": 2, + "v0": "coupled", + "recoded": "other" + }, + { + "id": "python-2018-12-governance-model-vote-pep-8016", + "i": 0, + "v0": "rank", + "recoded": "vote" + }, + { + "id": "python-2019-06-pep-602-annual-release-cycle", + "i": 5, + "v0": "supersede", + "recoded": "supersede" + }, + { + "id": "python-2023-05-pep-649-deferred-annotations-supersedes-563", + "i": 0, + "v0": "staged", + "recoded": "staged" + }, + { + "id": "rust-lang-team-2021-07-decision-process-tenets-and-dissent", + "i": 0, + "v0": "consensus-mechanics", + "recoded": "consensus-mechanics" + }, + { + "id": "rust-lang-team-2022-03-lang-roadmap-themes", + "i": 0, + "v0": "rank", + "recoded": "rank" + }, + { + "id": "rust-lang-team-2022-03-lang-roadmap-themes", + "i": 2, + "v0": "implicit", + "recoded": "implicit" + }, + { + "id": "rust-lang-team-2022-03-lang-roadmap-themes", + "i": 3, + "v0": "other", + "recoded": "other" + }, + { + "id": "rust-lang-team-2022-03-lang-roadmap-themes", + "i": 5, + "v0": "other", + "recoded": "other" + }, + { + "id": "rust-lang-team-2022-03-lint-policy-ownership", + "i": 4, + "v0": "evidence-not-check", + "recoded": "evidence-not-check" + }, + { + "id": "rust-lang-team-2022-03-lint-policy-ownership", + "i": 5, + "v0": "venue-split", + "recoded": "venue-split" + }, + { + "id": "rust-lang-team-2023-05-return-type-notation-spelling", + "i": 0, + "v0": "deferred", + "recoded": "deferred" + }, + { + "id": "rust-lang-team-2023-05-return-type-notation-spelling", + "i": 6, + "v0": "other", + "recoded": "reframe" + }, + { + "id": "tc39-2019-10-ecma402-casing-conventions", + "i": 3, + "v0": "staged", + "recoded": "staged" + }, + { + "id": "tc39-2019-10-proposal-disclosure-policy", + "i": 0, + "v0": "external-authority", + "recoded": "external-authority" + }, + { + "id": "tc39-2019-10-proposal-disclosure-policy", + "i": 1, + "v0": "reframe", + "recoded": "reframe" + }, + { + "id": "tc39-2019-10-proposal-disclosure-policy", + "i": 2, + "v0": "blocking", + "recoded": "delegate" + }, + { + "id": "tc39-2019-10-proposal-disclosure-policy", + "i": 5, + "v0": "deferred", + "recoded": "deferred" + }, + { + "id": "tc39-2019-12-dynamic-code-brand-checks-stage2-refused", + "i": 0, + "v0": "conditional", + "recoded": "conditional" + }, + { + "id": "tc39-2019-12-dynamic-code-brand-checks-stage2-refused", + "i": 1, + "v0": "consensus-mechanics", + "recoded": "consensus-mechanics" + }, + { + "id": "tc39-2019-12-dynamic-code-brand-checks-stage2-refused", + "i": 3, + "v0": "delegate", + "recoded": "approve" + }, + { + "id": "tc39-2020-04-weakrefs-finalizationregistry-cleanup", + "i": 1, + "v0": "external-authority", + "recoded": "delegate" + }, + { + "id": "tc39-2020-11-item-method-rename-to-at", + "i": 1, + "v0": "evidence-not-check", + "recoded": "evidence-not-check" + }, + { + "id": "tc39-2021-08-pipeline-operator-stage2-hack-style", + "i": 3, + "v0": "other", + "recoded": "conditional" + }, + { + "id": "tc39-2021-08-pipeline-operator-stage2-hack-style", + "i": 5, + "v0": "consensus-mechanics", + "recoded": "consensus-mechanics" + }, + { + "id": "tc39-2021-08-realm-constructor-shadowrealm-name", + "i": 2, + "v0": "other", + "recoded": "other" + }, + { + "id": "tc39-2021-08-realm-constructor-shadowrealm-name", + "i": 3, + "v0": "consensus-mechanics", + "recoded": "other" + }, + { + "id": "tc39-2021-08-realm-constructor-shadowrealm-name", + "i": 4, + "v0": "blocking", + "recoded": "blocking" + }, + { + "id": "tc39-2021-08-realm-constructor-shadowrealm-name", + "i": 5, + "v0": "consensus-mechanics", + "recoded": "consensus-mechanics" + }, + { + "id": "tc39-2022-01-stage3-normative-changes-need-consensus", + "i": 1, + "v0": "nested", + "recoded": "delegate" + }, + { + "id": "tc39-2022-01-stage3-normative-changes-need-consensus", + "i": 2, + "v0": "consensus-mechanics", + "recoded": "consensus-mechanics" + }, + { + "id": "tc39-2022-01-stage3-normative-changes-need-consensus", + "i": 3, + "v0": "venue-split", + "recoded": "other" + }, + { + "id": "tc39-2023-01-explicit-support-consensus-procedure", + "i": 1, + "v0": "consensus-mechanics", + "recoded": "consensus-mechanics" + }, + { + "id": "tc39-2023-01-explicit-support-consensus-procedure", + "i": 2, + "v0": "consensus-mechanics", + "recoded": "consensus-mechanics" + }, + { + "id": "tc39-2023-01-explicit-support-consensus-procedure", + "i": 4, + "v0": "consensus-mechanics", + "recoded": "consensus-mechanics" + }, + { + "id": "tc39-2023-05-array-grouping-static-methods", + "i": 0, + "v0": "provisional", + "recoded": "provisional" + }, + { + "id": "tc39-2023-11-operator-overloading-withdrawn", + "i": 0, + "v0": "moot", + "recoded": "other" + }, + { + "id": "tc39-2024-04-math-sumprecise-design-points", + "i": 4, + "v0": "consensus-mechanics", + "recoded": "crowd-signal" + }, + { + "id": "tc39-2025-09-iterator-windows-undersized-parameter", + "i": 2, + "v0": "other", + "recoded": "other" + }, + { + "id": "tc39-2025-09-iterator-windows-undersized-parameter", + "i": 4, + "v0": "convention", + "recoded": "convention" + } + ] +} diff --git a/evals/datasets/decisions-v1/analysis/thread-gap-tags.json b/evals/datasets/decisions-v1/analysis/thread-gap-tags.json new file mode 100644 index 000000000..c9013197d --- /dev/null +++ b/evals/datasets/decisions-v1/analysis/thread-gap-tags.json @@ -0,0 +1,804 @@ +{ + "appflowy-dark-mode-palette": [ + "crowd-signal", + "implicit", + "coupled", + "delegate" + ], + "ripgrep-long-line-handling": [ + "other", + "blocking", + "evidence-not-check", + "coupled" + ], + "ripgrep-hyperlink-support": [ + "revert", + "supersede", + "evidence-not-check", + "external-authority" + ], + "ripgrep-pager-flag": [ + "revert", + "crowd-signal", + "other", + "external-authority" + ], + "zoxide-fish-cd-call": [ + "revert", + "other" + ], + "uv-index-semantics": [ + "coupled", + "other", + "nested", + "venue-split" + ], + "atuin-classic-up-arrow": [ + "stale-idle", + "crowd-signal", + "evidence-not-check" + ], + "just-telemetry-mechanism": [ + "reframe", + "crowd-signal", + "stale-idle" + ], + "just-f-string-syntax": [ + "revert", + "nested", + "stale-idle", + "coupled" + ], + "just-parallel-mechanism": [ + "staged", + "evidence-not-check", + "crowd-signal", + "stale-idle" + ], + "crystal-io-first": [ + "crowd-signal", + "staged", + "consensus-mechanics" + ], + "delta-light-dark-detection": [ + "staged", + "stale-idle", + "supersede", + "partition" + ], + "deno-config-permissions": [ + "venue-split", + "delegate", + "rejected-content", + "other" + ], + "deno-rust-type-checker": [ + "nested", + "staged", + "external-authority", + "delegate" + ], + "deno-deny-flags-strict-check": [ + "venue-split", + "coupled", + "delegate" + ], + "deno-jupyter-display-hook": [ + "venue-split", + "staged", + "coupled", + "external-authority" + ], + "aspnetcore-mvc-in-shared-framework": [ + "evidence-not-check", + "other", + "nested" + ], + "eslint-formatting-rule-deprecation": [ + "venue-split", + "partition", + "staged", + "blocking", + "other" + ], + "eslint-native-ts-config-flag": [ + "conditional", + "other", + "consensus-mechanics" + ], + "eslint-adopt-codemod": [ + "venue-split", + "delegate", + "other", + "consequences" + ], + "excalidraw-chart-type-selection": [ + "venue-split", + "coupled", + "staged" + ], + "excalidraw-wheel-zoom": [ + "delegate", + "crowd-signal", + "evidence-not-check", + "evidence-not-check" + ], + "excalidraw-new-drawing-button": [ + "consensus-mechanics", + "crowd-signal", + "blocking", + "evidence-not-check" + ], + "excalidraw-handwritten-font": [ + "crowd-signal", + "venue-split", + "reframe", + "visual-comparison" + ], + "excalidraw-zen-mode-entry-exit": [ + "coupled", + "convention", + "evidence-not-check" + ], + "excalidraw-binding-default": [ + "evidence-not-check", + "staged", + "delegate" + ], + "excalidraw-export-split-labels": [ + "coupled", + "venue-split", + "nested", + "other" + ], + "excalidraw-embed-validator-api": [ + "implicit", + "coupled" + ], + "excalidraw-embed-alt-text": [ + "venue-split" + ], + "ghostty-broadcast-groups-design": [ + "implicit", + "coupled", + "blocking", + "approve" + ], + "ghostty-official-snap": [ + "implicit", + "conditional", + "blocking", + "venue-split", + "implicit" + ], + "godot-object-metadata": [ + "crowd-signal", + "nested", + "venue-split" + ], + "go-remove-binary-only-packages": [ + "consensus-mechanics", + "other", + "conditional", + "staged", + "venue-split" + ], + "go-try-builtin": [ + "crowd-signal", + "reframe", + "venue-split", + "venue-split", + "other", + "evidence-not-check", + "coupled" + ], + "go-slice-to-array": [ + "delegate", + "staged", + "consensus-mechanics", + "nested" + ], + "go-tb-output": [ + "delegate", + "coupled", + "consensus-mechanics", + "staged" + ], + "go-loopvar-per-iteration": [ + "evidence-not-check", + "venue-split", + "coupled", + "crowd-signal", + "venue-split", + "staged" + ], + "go-first-non-zero-helper": [ + "other", + "coupled", + "crowd-signal", + "nested", + "venue-split" + ], + "go-unique-generic-handle": [ + "venue-split", + "staged", + "coupled", + "other" + ], + "go-unique-naming": [ + "crowd-signal" + ], + "xterm-history-api-shape": [ + "venue-split", + "staged", + "coupled", + "consensus-mechanics", + "stale-idle" + ], + "helix-terminal-in-core": [ + "venue-split", + "implicit", + "crowd-signal", + "delegate", + "blocking" + ], + "helix-file-explorer-shape": [ + "coupled", + "delegate", + "crowd-signal", + "revert" + ], + "helix-explorer-design": [ + "supersede", + "stale-idle", + "staged", + "blocking" + ], + "helix-state-file-layout": [ + "stale-idle", + "evidence-not-check" + ], + "helix-state-scope": [ + "delegate", + "venue-split", + "stale-idle" + ], + "ha-floors-vs-nested-areas": [ + "ratify", + "nested", + "blocking", + "other" + ], + "ha-entity-name-editing": [ + "implicit", + "rejected-content", + "revert", + "delegate", + "venue-split" + ], + "ha-addons-to-apps": [ + "other", + "evidence-not-check", + "venue-split", + "reframe" + ], + "ha-template-binary-sensor-delays-ui": [ + "venue-split", + "other", + "crowd-signal", + "rejected-content", + "evidence-not-check" + ], + "home-assistant-network-interface-policy": [ + "reframe", + "evidence-not-check", + "reframe", + "stale-idle" + ], + "ha-numeric-state-unavailable": [ + "stale-idle", + "venue-split", + "other", + "other" + ], + "ha-device-ip-display": [ + "implicit", + "delegate", + "stale-idle", + "coupled" + ], + "home-assistant-lawn-mower-entity-model": [ + "venue-split", + "other", + "coupled", + "staged", + "consequences" + ], + "ha-event-entity-shape": [ + "venue-split", + "coupled", + "other", + "partition", + "consensus-mechanics" + ], + "home-assistant-fan-speed-model": [ + "reframe", + "staged", + "coupled", + "stale-idle" + ], + "ha-release-cycle-length": [ + "venue-split", + "stale-idle", + "reframe", + "evidence-not-check" + ], + "home-assistant-last-changed-persistence": [ + "staged", + "stale-idle", + "delegate", + "evidence-not-check", + "other" + ], + "ha-code-owner-required": [ + "other", + "supersede", + "consensus-mechanics" + ], + "home-assistant-choose-action-syntax": [ + "supersede", + "venue-split", + "coupled", + "other" + ], + "ha-zwave-driver-choice": [ + "venue-split", + "revert", + "consequences", + "other" + ], + "frontend-section-styling": [ + "venue-split", + "revert", + "crowd-signal", + "supersede" + ], + "ha-toggle-confirmation": [ + "staged", + "delegate", + "crowd-signal", + "evidence-not-check", + "supersede", + "consensus-mechanics" + ], + "lazygit-worktree-naming-location": [ + "staged", + "evidence-not-check", + "deferred", + "other" + ], + "fzf-default-opts-file": [ + "ratify", + "consequences", + "other" + ], + "fzf-preview-window": [ + "implicit", + "partition", + "visual-comparison", + "other" + ], + "logseq-hierarchy-placement": [ + "delegate", + "crowd-signal", + "venue-split", + "implicit" + ], + "mastodon-remote-local-timeline": [ + "moot", + "consensus-mechanics", + "other", + "crowd-signal" + ], + "mastodon-quote-posts-policy": [ + "supersede", + "delegate", + "other", + "crowd-signal", + "venue-split" + ], + "mastodon-text-formatting": [ + "venue-split", + "crowd-signal", + "delegate", + "revert", + "evidence-not-check" + ], + "mastodon-local-only-posts": [ + "reframe", + "supersede", + "evidence-not-check", + "deferred", + "crowd-signal" + ], + "typescript-node-esm-mode-shape": [ + "venue-split", + "other", + "supersede", + "staged" + ], + "vscode-scm-single-view-response": [ + "revert", + "crowd-signal", + "supersede", + "delegate" + ], + "vscode-centered-layout-scope-and-settings": [ + "crowd-signal", + "implicit", + "other", + "other" + ], + "vscode-focus-indicator-dim-vs-border": [ + "crowd-signal", + "evidence-not-check", + "staged", + "stale-idle" + ], + "vscode-dim-setting-shape-and-namespace": [ + "revert", + "supersede", + "coupled" + ], + "vscode-modern-ui-rollout": [ + "delegate", + "venue-split", + "crowd-signal", + "evidence-not-check", + "staged" + ], + "vscode-modern-ui-optin-rollout": [ + "delegate", + "evidence-not-check", + "revert", + "crowd-signal", + "other" + ], + "vscode-modern-ui-classic-removal": [ + "delegate", + "revert", + "crowd-signal", + "staged", + "evidence-not-check" + ], + "vscode-completion-item-label-shape": [ + "coupled", + "staged", + "stale-idle", + "blocking", + "visual-comparison" + ], + "vscode-quick-open-preview-behaviour": [ + "staged", + "venue-split", + "other", + "blocking", + "crowd-signal", + "implicit" + ], + "vscode-pinned-tab-sizing": [ + "coupled", + "revert", + "other", + "crowd-signal", + "nested", + "evidence-not-check" + ], + "vscode-fixed-width-tabs": [ + "evidence-not-check", + "reframe", + "coupled", + "other" + ], + "vscode-windows-titlebar-style": [ + "venue-split", + "blocking", + "coupled", + "delegate", + "crowd-signal" + ], + "node-yarn-vs-corepack": [ + "vote", + "venue-split", + "staged", + "delegate" + ], + "node-tsc-voting-method": [ + "coupled", + "nested" + ], + "node-primordials-error-path": [ + "vote", + "venue-split", + "partition", + "other", + "venue-split" + ], + "nodejs-primordials-policy": [ + "supersede", + "crowd-signal", + "staged", + "supersede", + "reframe" + ], + "node-incomingmessage-destroy": [ + "revert", + "implicit", + "approve", + "evidence-not-check" + ], + "node-rm-eperm-workaround": [ + "approve", + "evidence-not-check", + "evidence-not-check", + "implicit" + ], + "outline-document-tags": [ + "crowd-signal", + "delegate", + "other", + "deferred" + ], + "outline-raw-markdown-mode": [ + "conditional", + "consensus-mechanics", + "evidence-not-check" + ], + "outline-database-views": [ + "crowd-signal", + "reframe", + "other", + "supersede" + ], + "penpot-desktop-app-form": [ + "crowd-signal", + "coupled", + "venue-split", + "staged", + "implicit", + "crowd-signal" + ], + "pnpm-pre-post-scripts": [ + "revert", + "staged", + "crowd-signal", + "coupled", + "consequences" + ], + "pnpm-pm-strict-default": [ + "crowd-signal", + "venue-split", + "revert", + "external-authority" + ], + "prettier-final-newline": [ + "convention", + "crowd-signal", + "conditional", + "reframe" + ], + "prettier-generator-star-revert": [ + "consensus-mechanics", + "revert", + "consensus-mechanics" + ], + "prettier-nested-ternary-format": [ + "crowd-signal", + "provisional", + "other", + "convention", + "nested" + ], + "rfcs-abort-default-strategy": [ + "venue-split", + "supersede", + "nested", + "approve" + ], + "rfcs-crate-changelogs": [ + "venue-split", + "partition", + "approve", + "supersede", + "reframe" + ], + "rust-bool-option-methods": [ + "coupled", + "staged", + "supersede", + "approve" + ], + "rust-check-cfg-rfc": [ + "implicit", + "delegate", + "staged" + ], + "cargo-feature-doc-field": [ + "staged", + "coupled", + "venue-split", + "approve", + "delegate" + ], + "rfcs-derive-from-direction": [ + "staged", + "coupled", + "approve", + "consensus-mechanics", + "delegate" + ], + "rust-analyzer-attr-macro-input-fixup": [ + "external-authority", + "venue-split", + "staged", + "implicit" + ], + "ra-upstream-inlay-hints": [ + "external-authority", + "venue-split", + "coupled", + "blocking" + ], + "bat-builtin-pager-approach": [ + "venue-split", + "staged", + "conditional" + ], + "bat-debian-batcat-name": [ + "revert", + "external-authority", + "venue-split", + "conditional" + ], + "svelte-raw-snippet-functions": [ + "evidence-not-check", + "coupled", + "evidence-not-check", + "approve" + ], + "svelte-element-this-attr": [ + "venue-split", + "supersede", + "coupled", + "staged" + ], + "svelte-css-nesting-implementation": [ + "coupled", + "supersede", + "venue-split", + "evidence-not-check" + ], + "tailwind-variant-grouping": [ + "venue-split", + "delegate", + "conditional", + "implicit" + ], + "tailwindcss-prettier-strip-whitespace-default": [ + "revert", + "crowd-signal", + "venue-split", + "evidence-not-check" + ], + "tc39-reflection-syntax-shape": [ + "venue-split", + "supersede", + "consensus-mechanics", + "other" + ], + "uuid6-alt-text-encoding": [ + "supersede", + "venue-split", + "coupled", + "evidence-not-check", + "supersede" + ], + "nextjs-global-css-imports": [ + "supersede", + "staged", + "stale-idle", + "partition" + ], + "nextjs-page-file-and-dir-naming": [ + "crowd-signal", + "venue-split", + "coupled", + "implicit", + "other" + ], + "nextjs-auth-interrupts-api-shape": [ + "delegate", + "evidence-not-check", + "coupled", + "other", + "stale-idle" + ], + "nextjs-csp-approach": [ + "staged", + "supersede", + "venue-split", + "coupled" + ], + "vite-lib-externalize-lookup": [ + "stale-idle", + "evidence-not-check", + "deferred" + ], + "vite-eager-browserhash-merge": [ + "delegate", + "approve", + "blocking", + "evidence-not-check" + ], + "vite-node-worker-api": [ + "venue-split", + "staged", + "supersede" + ], + "astro-define-vars-scripts": [ + "staged", + "approve", + "coupled", + "supersede" + ], + "astro-scripts-only-when-rendered": [ + "evidence-not-check", + "stale-idle", + "venue-split", + "partition" + ], + "astro-inline-script-stage-2": [ + "staged", + "approve", + "supersede", + "stale-idle" + ], + "zed-commit-log-graph": [ + "venue-split", + "visual-comparison", + "crowd-signal", + "staged" + ], + "zed-git-panel-file-diff": [ + "vote", + "reframe", + "other" + ], + "zed-file-tree-refresh-approach": [ + "delegate", + "implicit", + "consensus-mechanics", + "conditional", + "crowd-signal" + ], + "zed-threads-sidebar-disable-option": [ + "venue-split", + "other", + "crowd-signal", + "evidence-not-check", + "nested" + ], + "zed-hide-title-bar": [ + "revert", + "venue-split", + "crowd-signal", + "blocking", + "delegate" + ], + "zed-smooth-scroll-trackpad": [ + "staged", + "other", + "evidence-not-check", + "delegate" + ] +} diff --git a/evals/datasets/decisions-v1/documents/convert_v0.py b/evals/datasets/decisions-v1/documents/convert_v0.py new file mode 100644 index 000000000..2683e2733 --- /dev/null +++ b/evals/datasets/decisions-v1/documents/convert_v0.py @@ -0,0 +1,157 @@ +"""Convert the cloud session's document records (corpus v0) to the decisions-v1 record format. + +Usage: python3 -I convert_v0.py PATH/TO/corpus-v0-tagged.jsonl + +Keeps only records read from complete files (fidelity.method == "git-file"), +excluding the chopin-v2 collector, whose threads decisions-v1 refetched in full. +Writes documents/decisions.jsonl. +""" + +import json +import pathlib +import re +import sys + +HERE = pathlib.Path(__file__).resolve().parent + +FLOWS = {"find": "find", "work in place": "work-in-place", "research/spike": "research-or-spike", + "ask teammate": "ask-teammate", "settle": "settle", "revisit/reopen": "revisit-reopen", + "build/pr": "build-pr", "reshape": "reshape"} + +# v0 tagged every gap; tags that match a decisions-v1 cluster use its name, the rest keep their own. +CLUSTER = { + "staged": "staged acceptance", + "coupled": "coupled decisions", + "venue-split": "off-platform decision", + "crowd-signal": "votes as evidence", + "revert": "reversal after shipping", + "supersede": "superseded or moved to another thread", + "stale-idle": "dormant or ownerless question", + "reframe": "reframing", + "evidence-not-check": "evidence beyond check", + "delegate": "delegated decider", + "external-authority": "external authority", + "nested": "nested follow-ups", + "consequences": "consequences and obligations", + "blocking": "blocking dependency", + "consensus-mechanics": "consensus mechanics", + "vote": "formal vote", + "rank": "ranked choice", + "convention": "convention scope", + "rejected-content": "rejected options content", + "conditional": "conditional decision", + "provisional": "provisional decision", + "deferred": "deferred", + "implicit": "implicit decision", + "moot": "moot question", + "approve": "approval roster", + "visual-comparison": "visual comparison", +} +# The same keyword split v0's analysis applied to its catch-all `other` tag. +OTHER = [ + ("hidden history", re.compile(r"hidden|deleted|collapsed|unloaded|load more|unaccounted|moderation|locked", re.I)), + ("deadline", re.compile(r"deadline|time pressure|urgency|release schedule|patience|called time|should have come earlier|shipping urgency|edition-boundary", re.I)), +] + + +# v0 phrases that repeat 8+ consecutive words of their source document; reworded so +# committed records stay paraphrase-only (documents/validate.py checks this). +REWORD = { + "the current combination of GitHub Actions, ArgoCD and Argo Workflows": + "today's mix of Argo Workflows, ArgoCD and GitHub Actions", + "safe to enable in new and upgraded clusters (dry-run, audit)": + "can be switched on safely for fresh or upgraded clusters (dry-run, audit)", +} + + +def reword(value): + if isinstance(value, str): + for old, new in REWORD.items(): + value = value.replace(old, new) + return value + if isinstance(value, list): + return [reword(v) for v in value] + if isinstance(value, dict): + return {k: reword(v) for k, v in value.items()} + return value + + +def decider(text): + t = (text or "").strip() + for prefix, value in (("consensus", "consensus"), ("vote", "vote"), ("bdfl", "BDFL"), + ("steering council", "vote"), ("named maintainer", "named maintainer"), + ("unclear", "unclear"), ("ecma-402 task group", "team"), ("team", "team")): + if t.lower().startswith(prefix): + return value + raise ValueError(f"Unmapped decider: {t!r}") + + +def cluster(tag, gap): + if tag == "other": + for name, rx in OTHER: + if rx.search(gap): + return name + return "other" + return CLUSTER[tag] + + +def convert(r): + fit = r["chopinFit"] + if len(r["gapTags"]) != len(fit["gaps"]): + raise ValueError(f"{r['id']}: gapTags and gaps differ in length") + flows = [] + for f in fit["fitsFlows"]: + name = FLOWS[re.sub(r"^F\d+\s+", "", f).strip().lower()] + if name not in flows: + flows.append(name) + participants = dict(r["participants"]) + original = participants["decider"] + participants["decider"] = decider(original) + notes = (r.get("notes") or "").strip() + if original != participants["decider"]: + notes = (notes + " " if notes else "") + f"Original decider wording (v0): {original}." + process = dict(r["process"]) + process["reactionsTotal"] = None + process["decidedOffPlatform"] = bool(r["process"].get("decidedOffPlatform", False)) + ordered_process = {k: process[k] for k in ("research", "spike", "poll", "escalation", + "subQuestions", "deferred", "reopened", "superseded", "durationDays", "comments", + "reactionsTotal", "decidedOffPlatform", "notableMoves")} + return { + "id": r["id"], + "caseId": None, + "collector": "cloud-git", + "sources": r["sources"], + "repo": r["repo"], + "domain": r["domain"], + "title": r["title"], + "question": r["question"], + "questionExplicit": r["questionExplicit"], + "context": r["context"], + "options": r["options"], + "criteria": r["criteria"], + "participants": participants, + "process": ordered_process, + "outcome": r["outcome"], + "chopinFit": { + "shape": fit["shape"], + "shapeWhy": fit["shapeWhy"], + "secondaryShape": fit.get("secondaryShape"), + "fitsFlows": flows, + "gaps": [f"{cluster(t, g)}: {g}" for t, g in zip(r["gapTags"], fit["gaps"])], + }, + "evalUse": r["evalUse"], + "fidelity": {"method": "git-file", "truncated": r["fidelity"]["truncated"]}, + "notes": notes, + } + + +def main(): + records = [json.loads(l) for l in pathlib.Path(sys.argv[1]).read_text().splitlines() if l.strip()] + keep = [r for r in records if r["fidelity"]["method"] == "git-file" and r["collector"] != "chopin-v2"] + out = [reword(convert(r)) for r in keep] + (HERE / "decisions.jsonl").write_text("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in out)) + print(f"{len(records)} v0 records, {len(out)} converted") + + +if __name__ == "__main__": + main() diff --git a/evals/datasets/decisions-v1/documents/decisions.jsonl b/evals/datasets/decisions-v1/documents/decisions.jsonl new file mode 100644 index 000000000..444d17e99 --- /dev/null +++ b/evals/datasets/decisions-v1/documents/decisions.jsonl @@ -0,0 +1,41 @@ +{"id": "eng-backstage-adr013-014-http-fetch-library", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/backstage/backstage/blob/master/docs/architecture-decisions/adr014-use-fetch.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}, {"url": "https://github.com/backstage/backstage/blob/master/docs/architecture-decisions/adr013-use-node-fetch.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}], "repo": "backstage/backstage", "domain": "dependency", "title": "HTTP fetching library for Node.js code: node-fetch, then native fetch", "question": "Which HTTP fetching approach should Node.js-side Backstage code standardise on: the third-party node-fetch package or the platform's built-in fetch?", "questionExplicit": false, "context": "ADR013 standardised backend code on node-fetch to avoid carrying several HTTP packages. The project later raised its minimum Node.js to 20, which ships a stable built-in fetch, and third-party fetch packages had shown problems on newer Node versions.", "options": [{"label": "Several HTTP packages in use (axios, got and others)", "proposedBy": "unrecorded", "fate": "rejected"}, {"label": "Standardise Node.js code on node-fetch (ADR013)", "proposedBy": "unrecorded", "fate": "rejected"}, {"label": "Standardise Node.js code on native fetch, typed as typeof fetch (ADR014)", "proposedBy": "unrecorded", "fate": "chosen"}], "criteria": ["fewer dependencies and lower support burden", "compatibility with newer Node.js versions", "test tooling compatibility (msw interception)"], "participants": {"count": null, "roles": ["maintainers"], "decider": "unclear"}, "process": {"research": "links", "spike": "none", "poll": false, "escalation": "none", "subQuestions": ["frontend code keeps using the framework fetch API reference (unchanged)", "isomorphic packages take an injectable fetch (unchanged)"], "deferred": false, "reopened": true, "superseded": true, "durationDays": null, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["reversal was triggered by a platform change (raised minimum Node.js), not by dissent", "the superseding ADR repeats the unchanged frontend and isomorphic guidance and changes only the Node.js clause", "old ADR kept in full with a banner pointing forward to its replacement"]}, "outcome": {"decided": true, "decision": "All code executed in Node.js (backend and CLIs) uses the built-in fetch; node-fetch and similar replacement packages are phased out gradually.", "how": "unclear", "rationale": "Node.js 20+ has a stable undici-based fetch, and third-party packages were causing issues on newer Node versions.", "dissent": null, "confidence": "implied", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "A single standard is picked among a few named libraries, per code context.", "secondaryShape": null, "fitsFlows": ["settle", "revisit-reopen"], "gaps": ["superseded or moved to another thread: Supersession as a first-class link: old record stays readable, marked no longer in force, with a forward pointer, and the new record cites the old. The design has Reopen (same question returns to open) but no Supersede that creates a new settled question pointing back at the old.", "provisional decision: Reversal trigger is an environmental change (minimum platform version raised), not a new argument. The record has 'What we knew at the time' but nothing like 'revisit when X changes' or a recorded watch condition.", "consequences and obligations: Consequences carry a follow-on obligation (msw 1.x based tests may break and need rewriting). The record's 'Built into' tracks PRs, not obligations created by the decision.", "convention scope: One decision with three scopes (Node.js code, frontend, isomorphic packages), only one of which changed. The choice table holds a single answer, not per-context answers where one cell is revised."]}, "evalUse": "Clean supersession pair with an external trigger and a consequences section that anticipates test breakage; no debate recorded.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "Shallow clone has no history, so decision dates and authors are unknown (null). Neither ADR names people or an approver, so participants.count is null. Counted because ADR014 sets native fetch against the previously adopted node-fetch and ADR013 sets one standard against multiple packages. The underlying discussion (issue linked in ADR014 context, not read) is not captured; the debate itself is not recorded."} +{"id": "eng-cloud-platform-adr012-021-036-048-multi-cluster", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/ministryofjustice/cloud-platform/blob/main/architecture-decision-record/036-multi-cluster.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}, {"url": "https://github.com/ministryofjustice/cloud-platform/blob/main/architecture-decision-record/021-Multi-cluster.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}, {"url": "https://github.com/ministryofjustice/cloud-platform/blob/main/architecture-decision-record/012-One-cluster-for-dev-staging-prod.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}, {"url": "https://github.com/ministryofjustice/cloud-platform/blob/main/architecture-decision-record/048-per-business-unit-clusters.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}], "repo": "ministryofjustice/cloud-platform", "domain": "architecture", "title": "One shared Kubernetes cluster versus several clusters for service-team apps", "question": "Should service teams' dev, staging and production apps all run on one Kubernetes cluster, or should the platform split them across several clusters, and along what line?", "questionExplicit": true, "context": "In 2019 the team chose one cluster for everything, citing lower overhead and avoiding drift between environments. By 2021 it saw scaling limits, a single point of failure, risky upgrades and large blast radius, and proposed multi-cluster without deciding. In 2022, after moving to EKS and namespace growth from 409 to 457, it adopted a live/non-live split; in 2026 it went further to clusters per business unit.", "options": [{"label": "Single cluster with namespaces separating dev/staging/prod (ADR012)", "proposedBy": "unrecorded", "fate": "rejected"}, {"label": "Multi-cluster, possibly with separate VPCs or AWS accounts per cluster (ADR021 proposal, decision left TBD)", "proposedBy": "unrecorded", "fate": "partly"}, {"label": "Two clusters split by environment type: production and non-production (ADR036)", "proposedBy": "unrecorded", "fate": "chosen"}, {"label": "A live and a non-live cluster per business unit (ADR048, 2026)", "proposedBy": "unrecorded", "fate": "chosen"}], "criteria": ["blast radius", "scaling and AWS per-account limits", "upgrade risk", "cost and resource efficiency", "drift between environments", "operational complexity"], "participants": {"count": null, "roles": ["platform team"], "decider": "team"}, "process": {"research": "links", "spike": "none", "poll": false, "escalation": "none", "subQuestions": ["how to move apps between clusters (kubecfg tokens, IP allow-lists)", "AWS account per cluster (a later ADR)", "whether users choose their own environment cluster"], "deferred": true, "reopened": true, "superseded": true, "durationDays": 493, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["ADR012 was annotated in place twice (June 2020, May 2021) while still marked Accepted", "ADR021 has a long argument but its decision section says only that it is to be decided; it was later marked superseded without ever being settled", "ADR036 adopts multi-cluster but starts with only the live/non-live split and lets users self-select initially", "ADR048 cites the noisy-neighbour and security gaps left by 036"]}, "outcome": {"decided": true, "decision": "Adopt multi-cluster, starting with separate production and non-production clusters (ADR036), later extended to a live and non-live cluster for each business unit (ADR048).", "how": "unclear", "rationale": "Growth in namespaces and platform maturity made the single-cluster costs outweigh its benefits; separate clusters reduce blast radius, single points of failure and upgrade risk, at higher cost and complexity.", "dissent": null, "confidence": "implied", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "Alternatives are architectures compared on stated criteria; the early record is closer to an explore.", "secondaryShape": "explore", "fitsFlows": ["revisit-reopen", "settle"], "gaps": ["dormant or ownerless question: A question left 'TBD' for about 16 months and then replaced rather than settled: ADR021 holds a full argument and an empty decision. The design has Open and Settled only. There is no 'superseded without ever being settled' or 'parked' state, and Reopen assumes a prior answer exists.", "superseded or moved to another thread: One underlying question has four numbered records over about seven years (012, 021, 036, 048). Reopen revisits the same question; this needs a lineage view across separately-numbered records showing who replaced which answer and why.", "superseded or moved to another thread: Annotations appended to a record that stays Accepted (the 2020 and 2021 updates in ADR012). There is no light 'note on a settled question' that does not reopen it.", "staged acceptance: Staged decision: adopt multi-cluster in principle, implement only the first split, leave user self-selection to be automated later. Settle yields one sentence and no 'step 1 of n' structure.", "provisional decision: Evidence is a dated snapshot ('at time of writing' namespace counts, comparison against Kubernetes large-cluster limits). 'What we knew' could hold it, but nothing flags it for refresh when the numbers change.", "consensus mechanics: Authority is the team, recorded as 'we decided', with no named person; the record's 'Settled by' face has nobody to show."]}, "evalUse": "Four-record chain on one question with an undecided intermediate record and in-place annotations; good for lineage and reopening tests.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "ADR012 is dated 01/06/2019; dates elsewhere in this repo are day-first and the ADR text mentions June 2020 as a later update, so it is read as 1 June 2019 (ambiguity noted). durationDays counts ADR021 (2021-05-11) to ADR036 (2022-09-16). No authors or approvers are named. The Google doc linked from ADR021 and the incident write-ups it cites were not read. Options were weighed inside ADR012 and 021 as pros and cons, not as a table."} +{"id": "eng-cloud-platform-adr018-019-ingress-controllers", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/ministryofjustice/cloud-platform/blob/main/architecture-decision-record/019-Shared-Ingress-Controllers.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}, {"url": "https://github.com/ministryofjustice/cloud-platform/blob/main/architecture-decision-record/018-Dedicated-Ingress-Controllers.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}], "repo": "ministryofjustice/cloud-platform", "domain": "infra", "title": "Dedicated ingress controller per namespace versus shared controller pool", "question": "How should the platform stop its single shared nginx ingress controller crash-looping as its config grows: one load balancer and controller per namespace, or a managed pool of shared controllers?", "questionExplicit": false, "context": "All ingresses fed one very large nginx config behind one load balancer, and several controller pods were crash-looping. ADR018 chose a dedicated controller per namespace. The migration was started, then abandoned when AWS hard limits on network load balancers blocked it; ADR019 proposed a shared design, and an update weeks later removed the dedicated controllers entirely after another AWS limit hit during an upgrade.", "options": [{"label": "Keep one default controller for every ingress", "proposedBy": "unrecorded", "fate": "rejected"}, {"label": "Dedicated load balancer and controller per namespace (ADR018)", "proposedBy": "unrecorded", "fate": "rejected"}, {"label": "Shared pool: default controller without the web application firewall, a managed pool with it, a few dedicated controllers left in place (ADR019)", "proposedBy": "unrecorded", "fate": "partly"}, {"label": "All ingresses on the default controller with firewall annotations removed (update note on ADR019)", "proposedBy": "unrecorded", "fate": "chosen"}], "criteria": ["reliability of the ingress layer", "AWS service limits", "hosting cost", "web application firewall coverage for production services"], "participants": {"count": null, "roles": ["platform team"], "decider": "team"}, "process": {"research": "links", "spike": "experiment", "poll": false, "escalation": "none", "subQuestions": ["how many firewall-enabled ingresses one controller can safely carry", "whether teams may define ingresses outside the environments repository"], "deferred": false, "reopened": true, "superseded": true, "durationDays": 7, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["a rollout of ADR018 was started and aborted when AWS network load balancer limits were hit", "ADR019 stays Proposed in the index while it was acted on", "a dated update appended to ADR019 reports that a second AWS limit led to removing all dedicated controllers"]}, "outcome": {"decided": true, "decision": "Dedicated per-namespace controllers were dropped; a shared design was proposed and then simplified so ingresses end up on the default controller without firewall annotations.", "how": "unclear", "rationale": "AWS hard limits (load balancers per VPC and zone, then security group rules) made the dedicated model impossible to operate.", "dissent": null, "confidence": "implied", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "Two architectures with concrete trade-offs, the second chosen after the first failed in practice.", "secondaryShape": null, "fitsFlows": ["research-or-spike", "revisit-reopen"], "gaps": ["reversal after shipping: Rejection came from implementation evidence (a quota nobody had checked), not from debate. The design has Spike (F3) before Settle; here the feasibility check happened after commitment, and there is no state for 'tried, failed, withdrawn' that preserves what the attempt taught.", "superseded or moved to another thread: The replacement record stays 'Proposed' although it was acted on, and its real current answer lives only in an appended update note six weeks later. A settled answer that drifts after settling needs an amendment event, not just Reopen.", "reversal after shipping: Incident reports are the evidence for the reversal. 'What we knew' lacks first-class incident links, and a reversal driven by an outage has no 'caused by' field.", "coupled decisions: Three sub-decisions bundled in the replacement (disable the firewall on default, run a firewall-enabled pool, leave some dedicated controllers): coupled questions settled together with one status.", "consequences and obligations: Cost consequences stated up front (more load balancers, pods and nodes); the record page has no section for accepted costs separate from rationale."]}, "evalUse": "Fast reversal driven by platform limits, with a still-Proposed replacement that was later amended; tests status semantics.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "Dates in ADR018/019 are written day-first (02/09/2020 and 09/09/2020), read here as 2 and 9 September 2020; the update note uses ISO 2020-10-14. durationDays is 018 to 019. No people are named; counted because both ADRs state options weighed and consequences. The linked Kubernetes issue and the incident log were not read."} +{"id": "eng-cloud-platform-adr003-040-ci-tooling", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/ministryofjustice/cloud-platform/blob/main/architecture-decision-record/040-use-github-actions.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}, {"url": "https://github.com/ministryofjustice/cloud-platform/blob/main/architecture-decision-record/003-Use-Concourse-CI.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}, {"url": "https://github.com/ministryofjustice/cloud-platform/blob/main/architecture-decision-record/042-use-modernisation-platform-accounts.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}], "repo": "ministryofjustice/cloud-platform", "domain": "tooling", "title": "Platform deployment pipelines: Jenkins, then Concourse, then GitHub Actions", "question": "What should run the platform team's deployment pipelines: its self-hosted Jenkins, a self-hosted Concourse, or GitHub Actions as used by the platform it is moving onto?", "questionExplicit": false, "context": "In 2018 the team replaced a fragile self-hosted Jenkins (about a week a month spent reviving it, recurring outages, blocked product teams) with Concourse on Kubernetes. In 2026, with the platform moving into the Modernisation Platform's accounts and pipelines, it decided to switch to GitHub Actions and retire Concourse and its management cluster.", "options": [{"label": "Keep self-hosted Jenkins", "proposedBy": "unrecorded", "fate": "rejected"}, {"label": "Self-hosted Concourse on Kubernetes (ADR003, 2018)", "proposedBy": "unrecorded", "fate": "rejected"}, {"label": "GitHub Actions, using the Modernisation Platform pipelines where possible (ADR040, 2026)", "proposedBy": "unrecorded", "fate": "chosen"}], "criteria": ["maintenance effort and downtime", "pipelines as code and cloud-native fit", "alignment with the platform being joined"], "participants": {"count": null, "roles": ["platform team"], "decider": "team"}, "process": {"research": "links", "spike": "prototype", "poll": false, "escalation": "none", "subQuestions": ["2018: deploy Concourse on AWS or Kubernetes (settled by a spike ticket, listed under Consequences)"], "deferred": false, "reopened": true, "superseded": true, "durationDays": 2837, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["2018 ADR lists tracker tickets as evidence and a struck-through success claim added later", "the 2018 ADR is still marked Accepted although the 2026 ADR retires Concourse", "the 2026 choice follows from a separate decision to move into another platform's accounts, so the tool choice is largely derived"]}, "outcome": {"decided": true, "decision": "Switch deployment pipelines to GitHub Actions and decommission Concourse and its management cluster once the move is complete.", "how": "unclear", "rationale": "The platform is moving to the Modernisation Platform, which already uses GitHub Actions, so one pipeline system replaces the self-managed Concourse.", "dissent": null, "confidence": "implied", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "Tool selection among named alternatives, repeated eight years apart.", "secondaryShape": null, "fitsFlows": ["revisit-reopen", "build-pr"], "gaps": ["superseded or moved to another thread: The reversal is not recorded on the original: ADR003 is still shown Accepted and ADR040 gives the Concourse background in its own context. The design assumes the reopened question and its record are the same object; here the contradiction is spread over two unlinked records.", "external authority: The real decider is an upstream strategy (join another platform). 'Decided elsewhere; we adopt' is not an authority type the design represents, and the options were effectively given.", "nested follow-ups: The 2018 record's Consequences is a running to-do list with sub-decisions ('deploy to Kubernetes') and ticket ids. Sub-decisions hide in consequences and evidence lives in an external tracker.", "consequences and obligations: Decommissioning is a stated future obligation tied to another migration finishing. 'Built into' tracks PRs, not 'retire X once Y completes'."]}, "evalUse": "Eight-year tool reversal where the old record was never updated; tests detection of an implied supersession.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "ADR003 date is 28/03/2018 (day-first read). ADR040 and ADR042 are both dated 2026-01-02. The ADR text says ADR039 relates to the move, but the file named 039 in the folder is about a network firewall and is dated 2025-09-16, so the cross-reference looks mismatched; not resolved here. Tracker tickets (CPT-xxx) were not accessible. Another team (alphagov/govuk-infrastructure ADR 0018, 2025) chose Concourse over Kargo in the opposite direction; see that record."} +{"id": "eng-modernisation-platform-adr004-019-core-languages", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/ministryofjustice/modernisation-platform/blob/main/architecture-decision-record/0019-use-bash-go-as-core-languages.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}, {"url": "https://github.com/ministryofjustice/modernisation-platform/blob/main/architecture-decision-record/0004-use-bash-node-python-as-core-languages.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}], "repo": "ministryofjustice/modernisation-platform", "domain": "tooling", "title": "Core programming languages for platform automation", "question": "Which languages should the platform team use for the scripts, GitHub Actions and Lambda functions it writes and maintains?", "questionExplicit": false, "context": "In 2020 the team picked bash, Node.js and Python based on popularity across the organisation and a skills poll. By 2022 it found those skills thinner than assumed, was already learning Go for its testing tool, and saw other teams switch to Go with good results.", "options": [{"label": "bash, Node.js and Python, each member choosing per problem (ADR004)", "proposedBy": "unrecorded", "fate": "rejected"}, {"label": "bash and Go, with Go the default for new scripts, Lambdas and tests (ADR019)", "proposedBy": "unrecorded", "fate": "chosen"}, {"label": "Java or .NET for Lambda", "proposedBy": "unrecorded", "fate": "rejected"}], "criteria": ["existing team skills", "Lambda cold-start time", "fit with the chosen testing tooling", "alignment with other teams", "energy use"], "participants": {"count": null, "roles": ["platform team"], "decider": "team"}, "process": {"research": "links", "spike": "none", "poll": true, "escalation": "none", "subQuestions": ["migrating existing scripts to Go over time (no plan)"], "deferred": false, "reopened": true, "superseded": true, "durationDays": 572, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["the 2020 decision rested on a team skills poll that tied between Node.js and Python (three each)", "the 2020 ADR itself says it can be revisited; the 2022 ADR does so without a stated trigger", "reversal justified by another team's positive experience"]}, "outcome": {"decided": true, "decision": "Use bash and Go as the core languages; new scripts, Lambdas and tests default to Go.", "how": "unclear", "rationale": "The team lacked deep Node.js and Python skills, was already learning Go for Terratest, and could learn from other MoJ teams on Go; Go also has good Lambda start times.", "dissent": null, "confidence": "implied", "reversibility": "stated-easy", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "Pick a language set against criteria; the earlier record shows a poll feeding the choice.", "secondaryShape": null, "fitsFlows": ["ask-teammate", "revisit-reopen"], "gaps": ["formal vote: A poll feeding a decision, tied 3-3, followed by a rule to let each member pick. The design has Ask and Takes (Agrees/Objects/Unsure) but no poll tally, no quorum, and no 'decide not to decide' outcome.", "provisional decision: Both ADRs say they can be revisited, but neither names a review date or trigger. The design has no 'revisit when' condition on a settled question.", "convention scope: A default-rule decision ('Go by default for new work') with legacy migration 'over time': a policy with scope and exceptions, not one sentence. No place for migration debt.", "superseded or moved to another thread: The 2020 rationale (team skills) turned out wrong; 'What we knew at the time' captures the belief but nothing later records that it was disproved."]}, "evalUse": "Team-level reversal with an explicit tied poll in the first record; good for 'team versus individual authority' cases.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "No individuals are named in either ADR. The poll counts (three for Node.js, three for Python) are given without saying how many people voted. Cold-start and energy claims are cited by link in ADR019; the linked articles were not read."} +{"id": "eng-modernisation-platform-adr002-003-access-management", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/ministryofjustice/modernisation-platform/blob/main/architecture-decision-record/0003-use-aws-sso.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}, {"url": "https://github.com/ministryofjustice/modernisation-platform/blob/main/architecture-decision-record/0002-use-iam-federated-access.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}, {"url": "https://github.com/ministryofjustice/modernisation-platform/blob/main/architecture-decision-record/0041-adopting-entra-id.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}], "repo": "ministryofjustice/modernisation-platform", "domain": "security", "title": "Identity and access management for the platform's AWS accounts", "question": "How should people get access to the platform's AWS accounts without the team running its own joiners, movers and leavers process: IAM federation through GitHub, or AWS SSO with automated provisioning from a corporate identity provider?", "questionExplicit": false, "context": "Many people across the department needed access. Five weeks apart, one ADR described federation with GitHub as the identity provider and was then marked Rejected; the next chose AWS SSO with SCIM provisioning. A 2026 ADR later adds the organisation's enterprise identity service because GitHub-only access had become unsuitable for non-developers.", "options": [{"label": "IAM federated access with GitHub as identity provider (ADR002)", "proposedBy": "unrecorded", "fate": "rejected"}, {"label": "AWS SSO with SCIM provisioning from a corporate identity provider (ADR003)", "proposedBy": "unrecorded", "fate": "chosen"}, {"label": "GitHub as identity provider behind SSO via Auth0", "proposedBy": "unrecorded", "fate": "partly"}], "criteria": ["automating joiners, movers and leavers", "central, organisation-level identity management", "multi-account access", "Terraform support", "alignment with other platform teams"], "participants": {"count": null, "roles": ["platform team"], "decider": "team"}, "process": {"research": "links", "spike": "none", "poll": false, "escalation": "none", "subQuestions": ["whether GitHub can still be the identity provider behind SSO"], "deferred": false, "reopened": true, "superseded": true, "durationDays": 36, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["ADR002's body reads like a decision but its status says Rejected; the reason for rejection appears only in ADR003", "ADR003 lists advantages and disadvantages relative to the rejected option", "a 2026 ADR says GitHub Teams had always been the access route, which the 2020 pair does not explain"]}, "outcome": {"decided": true, "decision": "Use AWS SSO with SCIM automated provisioning, managed at the organisation level, instead of IAM federated access.", "how": "unclear", "rationale": "Central, organisation-wide identity management with reusable permission sets across accounts outweighed losing alignment with two sister platforms and the lack of a Terraform provider at the time.", "dissent": "Trade-off acknowledged in the ADR: not aligned with the sister platform teams at that time; no named objector.", "confidence": "implied", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "Two options compared in a pros and cons list, one rejected and one accepted.", "secondaryShape": null, "fitsFlows": ["settle", "revisit-reopen"], "gaps": ["rejected options content: A rejected record whose body reads as a decision. The reason it was rejected sits only in the next record. 'What we considered' lives inside the winning record, so a rejected option's own consequences are lost.", "consequences and obligations: Per-option consequences are recorded as differences against the rejected option, a comparison matrix. The choice table holds cells, not long-form per-option advantages and disadvantages.", "external authority: Scope-of-authority criterion: a team choice that binds an organisation-level root account (SSO is run at the department's root). The design has no field for 'who else does this bind'.", "consequences and obligations: Joiners/movers/leavers control is a security-assurance concern. There is no way to mark a criterion as a control and attach an owner or risk.", "implicit decision: Practice later diverged from the decision (the 2026 ADR describes GitHub Teams as the long-standing route) without a reversal record; long-lived decisions drift silently."]}, "evalUse": "Consecutive ADRs flipping within five weeks, with a later third record changing the practice; tests option-carryover across records.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "No individuals named. Dates: 0002 2020-09-15, 0003 2020-10-21, 0041 2026-02-13. ADR041 is listed as an extra source because it contextualises later practice; its own decision (take Entra ID into service, with language and group-approval sub-decisions deferred) is a separate decision not recorded here."} +{"id": "eng-govuk-infrastructure-adr007-application-logging", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/alphagov/govuk-infrastructure/blob/main/docs/architecture/decisions/0007-use-fluentbit-elasticsearch-and-kibana-for-application-logs.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}], "repo": "alphagov/govuk-infrastructure", "domain": "infra", "title": "Application log collection and storage on the Kubernetes platform", "question": "How should application logs be collected and stored on the new Kubernetes platform: sidecar or node-level agent, Fluentd or Fluent Bit, and which store and UI?", "questionExplicit": false, "context": "Logging had to be a default service for all containers, with logs annotated by cluster metadata and searchable in a browser UI. The team expected it might later push logs to an organisation-wide store but wanted something running sooner.", "options": [{"label": "Logging sidecar in every pod", "proposedBy": "unrecorded", "fate": "rejected"}, {"label": "Node-level agent as a DaemonSet", "proposedBy": "unrecorded", "fate": "chosen"}, {"label": "Fluentd as the agent", "proposedBy": "unrecorded", "fate": "rejected"}, {"label": "Fluent Bit as the agent", "proposedBy": "unrecorded", "fate": "chosen"}, {"label": "Managed Elasticsearch with Kibana", "proposedBy": "unrecorded", "fate": "chosen"}, {"label": "Push to an organisation-wide central log store", "proposedBy": "unrecorded", "fate": "deferred"}], "criteria": ["logging as a default service for all containers", "Kubernetes metadata enrichment", "lightweight and fast agent", "plugin ecosystem", "time to a usable service"], "participants": {"count": null, "roles": ["platform engineers"], "decider": "team"}, "process": {"research": "links", "spike": "none", "poll": false, "escalation": "none", "subQuestions": ["control plane, node and kubelet logging (promised to a future ADR)", "how to authenticate users to Kibana"], "deferred": true, "reopened": false, "superseded": false, "durationDays": null, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["status later changed to Deprecated with the label 'no superseding ADR'", "scope restricted to application logs only"]}, "outcome": {"decided": true, "decision": "Run a Fluent Bit agent on each node and send logs to the managed Elasticsearch service with Kibana; later marked deprecated without a replacement record.", "how": "unclear", "rationale": "A node agent meets the default-for-all requirement; Fluent Bit is lighter and easy to swap with Fluentd; the managed service gives search UI quickly.", "dissent": null, "confidence": "implied", "reversibility": "stated-easy", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "Layered choices among named tools with trade-offs.", "secondaryShape": null, "fitsFlows": ["settle", "revisit-reopen"], "gaps": ["moot question: Deprecated with no successor: the decision was withdrawn rather than replaced, and the repo does not say what is now true. Settled and Reopen exist, but there is no 'retired, no replacement' state and a reader cannot find the current answer.", "nested follow-ups: A promised follow-up decision (control plane and node logs) does not appear among the later ADRs in the folder. Open child questions promised in prose are not tracked.", "conditional decision: Explicitly interim: chosen for now pending a possible organisation-wide store, a decision conditional on a future service owned by someone else.", "consequences and obligations: Must-do obligations in Consequences (restrict Kibana access, explore SSO options) without owner or question; the design cannot turn a consequence into a follow-up question.", "nested follow-ups: Two reversibility levels in one record: the agent is easy to swap, the store is not. Confidence is one value per record."]}, "evalUse": "Stack-selection ADR later deprecated without replacement; useful for status and 'current answer unknown' handling.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "Shallow clone: the deprecation date and who deprecated it are unknown (null). Status text is Deprecated (no superseding ADR). The ADR is dated 2021-08-16. No people named; options are weighed in prose so counted. The claim that no later ADR covers control plane logs is based on the 24 ADR titles in the folder, not on a full repo search."} +{"id": "eng-govuk-infrastructure-adr018-concourse-ci", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/alphagov/govuk-infrastructure/blob/main/docs/architecture/decisions/0018-use-concourse-ci.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}, {"url": "https://github.com/alphagov/govuk-infrastructure/blob/main/docs/architecture/decisions/0014-replace-terraform-cloud.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}], "repo": "alphagov/govuk-infrastructure", "domain": "tooling", "title": "Replace the CI/CD mix with Concourse after a Kargo versus Concourse trial", "question": "Which tool should replace today's mix of Argo Workflows, ArgoCD and GitHub Actions for building and deploying, after a security health check and the decision to leave Terraform Cloud?", "questionExplicit": false, "context": "An IT health check in 2024 flagged flaws in the deployment process, incidents had occurred, and the team had ongoing trouble with Argo Workflows. Leaving Terraform Cloud also created a need for something to apply Terraform automatically. A team design day picked two tools to trial.", "options": [{"label": "Keep GitHub Actions, ArgoCD and Argo Workflows", "proposedBy": "platform team", "fate": "rejected"}, {"label": "Kargo for multi-stage promotion", "proposedBy": "platform team", "fate": "rejected"}, {"label": "Concourse CI", "proposedBy": "platform team", "fate": "chosen"}, {"label": "Run Concourse on virtual machines in EC2", "proposedBy": "platform team", "fate": "rejected"}], "criteria": ["fixing the health-check security findings", "reliability", "documented, non-commercially-gated features", "fit with a Kubernetes-only operating model", "learning curve"], "participants": {"count": null, "roles": ["platform engineering team"], "decider": "team"}, "process": {"research": "links", "spike": "prototype", "poll": false, "escalation": "none", "subQuestions": ["in what architecture to deploy Concourse (later ADR)", "how to write pipelines (later ADR)"], "deferred": true, "reopened": false, "superseded": false, "durationDays": null, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["a team design day chose which two tools to trial", "each tool was prototyped on mock staging and production clusters by replaying the existing flow", "Kargo was dropped because the feature needed was left undocumented by its maintainers and a workaround defeated its validation", "findings were played back to the team before the decision"]}, "outcome": {"decided": true, "decision": "Pursue Concourse as the CI/CD tool, staying off self-managed virtual machines, with architecture and pipeline-authoring questions left to later ADRs.", "how": "unclear", "rationale": "Trialling found no blocking problems with Concourse, only a learning curve, while Kargo needed an undocumented feature; the team accepts trading a well-understood but fragile system for a simpler one.", "dissent": null, "confidence": "implied", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "Two tools prototyped and compared, one chosen.", "secondaryShape": "check", "fitsFlows": ["research-or-spike", "settle"], "gaps": ["staged acceptance: Staged decision: the tool is chosen and two named follow-up questions are deliberately left to future records. The design can make child questions but not 'settled with named open follow-ups linked to this answer'.", "evidence beyond check: Two parallel spikes, each with a verdict and an abandoned workaround, feeding option rows. F3 has one spike per question; per-option evidence is not modelled.", "external authority: Decision driven by an external audit finding whose details are deliberately omitted from the record: the rationale is partly confidential, so a record needs 'driver: audit finding' without the findings.", "coupled decisions: Coupled with another decision (leaving Terraform Cloud) that supplies part of the need: cross-question dependencies, one answer influencing another's criteria.", "consequences and obligations: Explicit statement of a temporary double-maintenance cost during a staged migration; consequences are over time, not static."]}, "evalUse": "Spike-driven tool selection with abandoned alternative, compliance driver and explicit follow-up ADRs.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "ADR dated 2025-08-12. ADR0014 (replace Terraform Cloud with S3 state, dated 2025-06-10) is an extra source: it states Accepted while its text says the ADR will move to accepted after migration, and defers a runner choice to a spike; not recorded as its own decision. File dates in the folder are not monotonic with ADR numbers (0015-0017, 0021, 0023, 0024 carry 2026 dates, one later than the clone date), so date reliability is uncertain. No individuals named."} +{"id": "eng-govuk-infrastructure-adr019-restructure-cicd", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/alphagov/govuk-infrastructure/blob/main/docs/architecture/decisions/0019-restructure-cicd.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}, {"url": "https://github.com/alphagov/govuk-infrastructure/blob/main/docs/architecture/decisions/0018-use-concourse-ci.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}], "repo": "alphagov/govuk-infrastructure", "domain": "architecture", "title": "Where Concourse runs and how artifacts are built and promoted", "question": "Where should Concourse run and how should artifacts be built, stored and promoted so deployments are secure, reproducible and visible across environments?", "questionExplicit": false, "context": "After choosing Concourse, the team spent roughly June to October 2025 on documents, conversations and meetings, and finalised the structure in team design sessions on 15 October 2025. Drivers were the 2024 health-check findings, user research with application teams and the move off Terraform Cloud.", "options": [{"label": "Concourse on EC2 instances", "proposedBy": "platform team", "fate": "rejected"}, {"label": "One Concourse on Kubernetes in a central management cluster", "proposedBy": "platform team", "fate": "partly"}, {"label": "A Concourse inside each environment's cluster", "proposedBy": "platform team", "fate": "partly"}, {"label": "Separate infrastructure and application Concourses per environment", "proposedBy": "platform team", "fate": "rejected"}, {"label": "One Concourse on one cluster holding all environments (raised during the design day; ADR gives no reason it was not taken)", "proposedBy": "platform team", "fate": "rejected"}], "criteria": ["security boundaries between environments", "single point of failure", "maintenance burden", "cross-environment visibility for application teams", "reproducible, build-once artifacts"], "participants": {"count": null, "roles": ["platform engineering team"], "decider": "team"}, "process": {"research": "links", "spike": "none", "poll": false, "escalation": "none", "subQuestions": ["whether to version rendered manifests in Git", "which YAML-generation tool to use (later ADR)"], "deferred": false, "reopened": false, "superseded": false, "durationDays": null, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["decision made in a team design session after months of side documents", "a fifth option was raised on the day and discussed briefly", "a build-once-and-promote rule was derived from flaky Rails asset precompilation", "manifest versioning decided as a secondary outcome of the same session"]}, "outcome": {"decided": true, "decision": "A blend of two options: a Concourse in each environment that deploys only there, plus a central 'tools' Concourse that builds artifacts once and cannot reach other accounts; rendered manifests will be versioned in Git.", "how": "consensus", "rationale": "Keeps environment separation and avoids a privileged management cluster, while building artifacts once on dedicated fast machines avoids unreliable rebuilds.", "dissent": null, "confidence": "implied", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "Five architectures compared, with a blended outcome.", "secondaryShape": null, "fitsFlows": ["settle", "build-pr"], "gaps": ["off-platform decision: Decided in a synchronous team session: a group reached it together. The design models asking individuals for takes, not a group session with N deciders and an agreed output.", "off-platform decision: Inputs are three external documents (principles, platform vision, deployment options) not in the repo. Evidence off-platform needs durable capture or the record will point at links that rot.", "rejected options content: A late option with no recorded reason for non-adoption: an option whose fate has no rationale.", "coupled decisions: Blended outcome of two rejected-in-part options, plus a derived requirement (build once) and a coupled second decision (versioned manifests). Settle yields one answer; coupled questions and hybrid choices have no structure.", "consequences and obligations: Consequences describe a year-long migration with a mixed state, accumulating debt and a risk of shelving the work. Settled records need an 'in flight' status and risk, not just Built into.", "convention scope: Reusable acceptance criteria ('core principles of secure CI/CD') are meant to judge future changes; criteria that outlive a single question are not representable.", "consequences and obligations: Compliance: driven by an external security assessment; consequence statement claims it addresses the most critical findings, but evidence lives outside the record."]}, "evalUse": "Group-session decision with five options, hybrid outcome and a long consequences section.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "ADR dated 2025-10-16, one day after the design session. The Google documents it cites were not read. The ADR says most debate happened June-October 2025 across several places; none of that is captured, so durationDays is null. Count of participants unknown (the ADR names none). The 'consensus' label is my reading of 'the team landed on'; the ADR does not use that word."} +{"id": "eng-modernisation-platform-adr032-ncsc-pdns", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/ministryofjustice/modernisation-platform/blob/main/architecture-decision-record/0032-ncsc-pdns-not-at-platform-level.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}], "repo": "ministryofjustice/modernisation-platform", "domain": "security", "title": "Protective DNS from the national cyber security agency versus DNS firewall at platform level", "question": "Should the platform implement the National Cyber Security Centre's protective DNS service for its customers, or use AWS Route 53 Resolver DNS Firewall?", "questionExplicit": true, "context": "The government protective DNS service was suggested as a way to improve customers' security. The team investigated it under an issue and compared it with a cloud-native firewall.", "options": [{"label": "Implement NCSC protective DNS", "proposedBy": "unrecorded (suggested to the team)", "fate": "rejected"}, {"label": "Implement AWS Route 53 Resolver DNS Firewall", "proposedBy": "unrecorded", "fate": "chosen"}], "criteria": ["cost", "provenance from a trusted government partner", "fit with cloud-native architecture", "manageable as code", "ability to apply at platform level", "portability"], "participants": {"count": null, "roles": ["platform team"], "decider": "team"}, "process": {"research": "links", "spike": "none", "poll": false, "escalation": "none", "subQuestions": ["scoping the DNS firewall implementation"], "deferred": false, "reopened": false, "superseded": false, "durationDays": null, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["investigation tracked in a GitHub issue (not read)", "outcome is a rejection plus a substitute to be scoped"]}, "outcome": {"decided": true, "decision": "Do not implement the national protective DNS at platform level; scope the AWS DNS firewall instead.", "how": "unclear", "rationale": "The national service is geared to organisation-wide, on-premise-style deployment and cannot be managed as code, while the cloud firewall gives comparable protection at platform level.", "dissent": null, "confidence": "implied", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "Two options with explicit pros and cons; outcome is accept one, decline the other.", "secondaryShape": null, "fitsFlows": ["settle"], "gaps": ["nested follow-ups: Outcome is 'no to a proposal from a trusted authority on fit grounds, and here is the substitute to scope'. The settle result is a negative plus a follow-up question, not a chosen cell; the follow-up (scope the firewall) should be spawned automatically.", "consequences and obligations: Provenance and assurance (a government-supplied service) as a criterion. The design has no way to flag compliance-flavoured criteria or note that the decision touches national security guidance.", "rejected options content: Pros and cons per option are the main artefact in long-form bullets, not table cells.", "other: Record identity is fragile: the file is numbered 0032 but its heading says 30, so ids cannot be trusted as typed by hand.", "off-platform decision: Investigation is tracked in a separate issue; the debate itself is not in the ADR, so the decision looks like it was made by writing rather than by discussion."]}, "evalUse": "Government security-service rejection with a substitute, compact options section; tests negative outcomes and spawned follow-ups.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "Dated 2024-07-10. Counted because the ADR lists two options with pros and cons. No people named. The investigation issue (#6121) was not read. Heading numbering mismatch (30 versus file 0032) observed as is."} +{"id": "eng-modernisation-platform-adr043-s3-encryption-scp", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/ministryofjustice/modernisation-platform/blob/main/architecture-decision-record/0043-enforce-s3-kms-encryption-with-scp-guardrails.md", "kind": "adr", "retrievedAt": "2026-10-18T05:45:00Z"}], "repo": "ministryofjustice/modernisation-platform", "domain": "security", "title": "Organisation-wide guardrails for S3 encryption, staged rollout", "question": "Which service control policies should enforce stronger S3 encryption across member accounts, and how should they be rolled out without disrupting existing workloads?", "questionExplicit": false, "context": "The team wanted stronger organisation-level control over S3 encryption. Testing in the policy repository showed which controls were reliable (object-write conditions) and which were not, and corrected some IAM action names. A past broad policy attachment had caused disruption, so the ADR prefers a pilot-first rollout.", "options": [{"label": "Deny setting bucket default encryption to the basic algorithm", "proposedBy": "unrecorded", "fate": "chosen"}, {"label": "Deny object writes that request the basic algorithm, applied broadly at OU level", "proposedBy": "unrecorded", "fate": "deferred"}, {"label": "Require a specific customer-managed key in conditions", "proposedBy": "unrecorded", "fate": "deferred"}, {"label": "Block customer-provided-key encryption as complementary hardening", "proposedBy": "unrecorded", "fate": "partly"}], "criteria": ["reliability of each control proven by tests", "blast radius", "compatibility with legacy workloads", "medium-term KMS-centric target"], "participants": {"count": null, "roles": ["platform team"], "decider": "unclear"}, "process": {"research": "benchmarks", "spike": "experiment", "poll": false, "escalation": "none", "subQuestions": ["when to require KMS keys (after pilot)", "whether to apply object-level denies broadly"], "deferred": true, "reopened": false, "superseded": false, "durationDays": 174, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["policy tests showed some assumed action names were invalid", "a previous broad attachment caused an incident, shaping a pilot-first rollout", "the Proposed status was unchanged at clone time, about 174 days after the ADR's date"]}, "outcome": {"decided": false, "decision": "Proposed: introduce a limited baseline policy denying explicit use of the basic algorithm, pilot it on a dedicated OU, and defer stricter KMS-only conditions.", "how": "unclear", "rationale": "Start with the controls the tests proved reliable and keep blast radius low while legacy workloads still depend on default encryption.", "dissent": null, "confidence": "stated", "reversibility": "stated-easy", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "Several candidate controls compared, most deferred pending pilot evidence.", "secondaryShape": "check", "fitsFlows": ["research-or-spike", "settle"], "gaps": ["dormant or ownerless question: Proposed for about six months with a fully written decision, pilot plan and rollback criteria. The design has Open and Settled only: a written-up decision awaiting approval is neither, and nothing says who is being waited on or flags that it has gone stale.", "conditional decision: Decision conditional on pilot evidence: attach first to a pilot OU, watch, widen in steps, with pre-approved detach criteria. Settle cannot carry gates or rollout conditions.", "reframing: Test findings corrected the question's own premises (which IAM action names are valid, what 'unencrypted' means in S3). The design treats research as outside the card; here research rewrote the question.", "consequences and obligations: Incident-informed caution as a criterion (previous broad attachment disrupted the platform) with no incident link field.", "staged acceptance: Security-compliance control deliberately incomplete at first (explicit basic algorithm blocked, KMS-only later): a deliberately partial settle."]}, "evalUse": "Long-lived Proposed ADR with phased rollout and conditions; tests status handling and conditional decisions.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "ADR date 2026-04-27, status Proposed at clone time 2026-10-18 (174 days). decided is false because the status is not accepted. Authors and approvers not named; participants.decider unclear. 'research: benchmarks' reflects policy tests run by the team, not performance benchmarks."} +{"id": "eng-k8s-kep2221-remove-dockershim", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://raw.githubusercontent.com/kubernetes/enhancements/master/keps/sig-node/2221-remove-dockershim/README.md", "kind": "proposal", "retrievedAt": "2026-10-18T05:45:00Z"}, {"url": "https://raw.githubusercontent.com/kubernetes/enhancements/master/keps/sig-node/2221-remove-dockershim/kep.yaml", "kind": "proposal", "retrievedAt": "2026-10-18T05:45:00Z"}], "repo": "kubernetes/enhancements", "domain": "compatibility", "title": "Deprecate and remove the built-in Docker runtime shim from kubelet", "question": "Should Kubernetes deprecate and remove the in-tree Docker container runtime shim from kubelet, leaving Docker support to an external adapter?", "questionExplicit": false, "context": "The Docker shim lives in kubelet and is tied to its lifecycle, creating maintenance burden for developers and administrators when runtime issues arise, and inconsistent integrations. The proposal lists pros and cons, sets removal criteria and a multi-release plan.", "options": [{"label": "Keep the shim in-tree", "proposedBy": "unrecorded", "fate": "rejected"}, {"label": "Deprecate in 1.20, then release kubelet without it (target 1.24) with an external adapter maintained by interested parties", "proposedBy": "@resouer, @dims (authors)", "fate": "chosen"}, {"label": "Ship builds without the shim by default while keeping code in the tree for a while", "proposedBy": "unrecorded", "fate": "partly"}], "criteria": ["maintenance burden", "CRI neutrality for runtimes", "number of affected users", "CRI API maturity", "Windows scenarios", "test infrastructure readiness"], "participants": {"count": 7, "roles": ["authors", "reviewers", "approvers"], "decider": "team"}, "process": {"research": "links", "spike": "none", "poll": false, "escalation": "none", "subQuestions": ["graduating the CRI API (separate KEP)", "out-of-tree Docker adapter ownership", "Windows support on other runtimes"], "deferred": false, "reopened": false, "superseded": false, "durationDays": null, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["Drawbacks and Alternatives sections say none while the Motivation weighs pros and cons", "removal gated on written criteria across several releases", "user-facing deprecation FAQ and kubelet warning shipped in 1.20 before removal"]}, "outcome": {"decided": true, "decision": "Deprecate the shim in 1.20 and remove it from kubelet by 1.24 once the criteria are met; Docker users move to an external adapter.", "how": "maintainer-decision", "rationale": "Docker should be one runtime among many; removing the shim reduces kubelet maintenance and lets the community move faster on runtime features.", "dissent": "Cons recorded: deployment pain from an extra binary, many affected users, CRI still alpha. No named objector in the KEP.", "confidence": "implied", "reversibility": "stated-hard", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "A removal with alternatives limited to 'keep'; the weighing sits in prose.", "secondaryShape": null, "fitsFlows": ["settle", "build-pr"], "gaps": ["conditional decision: Multi-release plan with criteria-gated steps (remove only once CRI is GA, e2e tests ready, etc.). The design has no timeline or gate checklist nor tracking of whether gates are met.", "approval roster: Authors, reviewers and approvers named separately with SIG roles; the decider is a pair of approvers, not one 'Settled by' face.", "consequences and obligations: Communication steps (migration guide, FAQ, warning) are part of the decision itself; the record has no 'announce' stage.", "rejected options content: The weighing lives under Motivation while Alternatives says none, so a tool must find the options in prose rather than a table.", "consequences and obligations: Large, unmeasured user impact and a rollback marked unsupported: a hard-to-reverse decision, while the record's confidence is one 'easy to change' line.", "staged acceptance: Change of status from proposal to implemented is only visible in a history list and YAML fields."]}, "evalUse": "Deprecation-and-removal KEP with explicit removal criteria; weighing in prose, not in the Alternatives section.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "Read in full from raw.githubusercontent.com (kubernetes/enhancements master), not a clone, because the repo is large; the text is complete, not a summary. participants.count 7 = 2 authors + 5 reviewers named in kep.yaml; approvers (2) overlap the reviewer list. The KEP lists Drawbacks 'None' and Alternatives 'None' although cons are listed in Motivation. Mailing-list discussion linked in the KEP was not read. Implementation History dates are ambiguous (02/03/2022); durationDays left null."} +{"id": "eng-k8s-kep2579-pod-security-admission", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://raw.githubusercontent.com/kubernetes/enhancements/master/keps/sig-auth/2579-psp-replacement/README.md", "kind": "proposal", "retrievedAt": "2026-10-18T05:45:00Z"}, {"url": "https://raw.githubusercontent.com/kubernetes/enhancements/master/keps/sig-auth/2579-psp-replacement/kep.yaml", "kind": "proposal", "retrievedAt": "2026-10-18T05:45:00Z"}], "repo": "kubernetes/enhancements", "domain": "security", "title": "Replace PodSecurityPolicy with a built-in admission controller", "question": "What should replace PodSecurityPolicy so Kubernetes can still limit privilege escalation through permission to create pods, given PodSecurityPolicy was deprecated instead of promoted to general availability?", "questionExplicit": false, "context": "PodSecurityPolicy had an awkward authorisation model, failed closed with no dry-run, and had an unbounded inconsistent API. The authors still wanted a built-in default control and proposed a validating admission plugin enforcing the Pod Security Standards through namespace labels.", "options": [{"label": "Promote PodSecurityPolicy to general availability", "proposedBy": "unrecorded", "fate": "rejected"}, {"label": "New built-in validating admission controller enforcing the Pod Security Standards via namespace labels", "proposedBy": "@tallclair, @liggitt (authors)", "fate": "chosen"}, {"label": "Leave replacement entirely to third-party policy engines", "proposedBy": "unrecorded", "fate": "rejected"}], "criteria": ["validating only, no mutation", "can be switched on safely for fresh or upgraded clusters (dry-run, audit)", "in-tree", "Windows future support", "extensibility", "ease of use"], "participants": {"count": 11, "roles": ["authors", "SIG reviewers", "approvers"], "decider": "team"}, "process": {"research": "links", "spike": "feature-flag", "poll": false, "escalation": "none", "subQuestions": ["Windows support", "automated migration tooling", "default baseline for unlabeled namespaces"], "deferred": true, "reopened": false, "superseded": false, "durationDays": null, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["initial proposal provisionally accepted from a design document before the KEP merged", "scope deliberately narrowed to avoid suppressing third-party policy engines and to avoid exemption creep", "Drawbacks section just points to Risks; Alternatives point to three external documents", "released behind a feature gate in alpha, beta and stable stages"]}, "outcome": {"decided": true, "decision": "Add the built-in Pod Security admission controller (namespace-label policies in enforce, audit and warn modes) to replace PodSecurityPolicy, with a published migration path.", "how": "maintainer-decision", "rationale": "Keeps an out-of-the-box defence against privilege escalation while fixing PSP's rollout and authorisation problems.", "dissent": null, "confidence": "implied", "reversibility": "stated-easy", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "Replacement design chosen against named alternatives, with strong requirements.", "secondaryShape": null, "fitsFlows": ["settle", "build-pr"], "gaps": ["superseded or moved to another thread: A replacement decision that follows an earlier settled one (deprecate rather than GA). The new record needs a 'replaces' link to something outside the plan, plus a migration path as part of the answer.", "approval roster: Sign-off needs named representatives from several teams (Windows, node, instrumentation, auth). That is a role-based approval roster, not 'ask a teammate for a take'.", "off-platform decision: Alternatives are in three off-repo design documents; they must be imported or the record has no 'what we considered'.", "consequences and obligations: A criterion about ecosystem effects: risk that a built-in standard becomes a de facto compliance requirement and suppresses third-party tools. The design has no 'who else is affected' field.", "staged acceptance: Staged acceptance: accepted in principle from a design doc, then alpha, beta and stable with a feature gate. Maturity stages and gate criteria are not part of Settled or Built into.", "nested follow-ups: Disabling the plugin is easy but removing PSP is not; reversibility differs for two actions in one record."]}, "evalUse": "Security-policy replacement KEP with explicit requirements, risks and staged graduation.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "Read in full from raw.githubusercontent.com (kubernetes/enhancements master), not a clone; text complete. participants.count 11 = authors (2), reviewers (4) and approvers (5) named in kep.yaml. The three alternatives documents were not read. Option 'leave replacement to third parties' is inferred from the ecosystem-suppression discussion and the goal of a built-in control; the KEP does not list it as a formal option. Calendar dates: provisional acceptance 2021-03-16, alpha 2021-08-04, beta release 2021-11-03; stable date not in file."} +{"id": "eng-nodejs-tsc-2026-ai-contribution-policy", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/nodejs/TSC/blob/main/meetings/2026-04-01.md", "kind": "minutes", "retrievedAt": "2026-10-18T05:45:00Z"}, {"url": "https://github.com/nodejs/TSC/blob/main/meetings/2026-03-18.md", "kind": "minutes", "retrievedAt": "2026-10-18T05:45:00Z"}, {"url": "https://github.com/nodejs/TSC/blob/main/meetings/2026-05-06.md", "kind": "minutes", "retrievedAt": "2026-10-18T05:45:00Z"}], "repo": "nodejs/TSC", "domain": "process-governance", "title": "Project policy on AI-assisted contributions", "question": "Should the project adopt contributor guidelines that permit AI-assisted contributions with disclosure and contributor responsibility, reject AI-generated contributions outright, or rely on existing policies?", "questionExplicit": true, "context": "A guidelines pull request and a tracking issue were on the committee agenda. The foundation's board separately approved a policy on AI coding assistants. The committee scheduled a live discussion for 1 April and a vote to follow.", "options": [{"label": "Adopt AI contribution guidelines in the contributing document, in line with the foundation policy", "proposedBy": "unrecorded (shepherded by @mcollina)", "fate": "deferred"}, {"label": "Reject AI-generated contributions entirely", "proposedBy": "@indutny (guest, emeritus)", "fate": "deferred"}, {"label": "Take no new action; existing contribution rules and the foundation policy apply", "proposedBy": "unrecorded", "fate": "deferred"}], "criteria": ["code quality and review burden", "ethics and attribution", "enforceability", "welcoming first-time contributors", "consistency with the foundation board's policy"], "participants": {"count": 19, "roles": ["voting members", "regular member", "guests incl. emeritus and CPC rep"], "decider": "vote"}, "process": {"research": "links", "spike": "none", "poll": false, "escalation": "vote", "subQuestions": ["whether existing policies already cover AI-assisted work", "whether disclosure can be enforced", "a dependent pull request waiting on this policy"], "deferred": true, "reopened": false, "superseded": false, "durationDays": 49, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["live discussion scheduled for a time friendly to US-timezone members, vote to follow", "foundation board approved its own policy before the committee voted", "a former committee member argued an ethical case against any AI use and ran a petition", "by 6 May the guidelines PR still lacked fresh approvals and the dissenter was to be asked whether he still objected", "other work (a virtual file system PR) was explicitly waiting for the policy to land"]}, "outcome": {"decided": false, "decision": "No outcome in the minutes read: as of 2026-05-06 the committee was still collecting re-reviews on the guidelines PR; the vote result is not in these minutes.", "how": "unclear", "rationale": null, "dissent": "One participant opposed accepting AI-assisted contributions at all, on grounds of attribution, ethics, and that reviewing plausible-looking generated code is an audit rather than a review; he argued that merely shifting responsibility to the contributor is not sufficient.", "confidence": "none", "reversibility": "unknown", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "Policy options exist but the dispute is about values, not the options.", "secondaryShape": "explore", "fitsFlows": ["ask-teammate", "settle", "revisit-reopen"], "gaps": ["formal vote: A formal vote among about 16 voting members after a scheduled live discussion. The design has Ask and Takes (Agrees/Objects/Unsure) for a few teammates, not an announced vote with a tally, quorum and a scheduled discussion before it.", "external authority: An absent authority constrains the options: the foundation board approved its policy first, so the committee chooses whether to add to an inherited rule.", "consensus mechanics: Disagreement on values and ethics that evidence cannot resolve. Research and Spike do not apply; a persistent blocking objection needs its own representation beyond 'Objects'.", "off-platform decision: One question across three meetings, an issue and a PR (policy vote, guidelines document, comms). The design assumes a single thread and has no cross-meeting continuation.", "blocking dependency: A blocking dependency: another PR waits on this decision. F7 covers handoff after settling, not 'blocked until settled' links in other threads.", "other: 'Do nothing' with the foundation policy applying by default is a real option and the dissenter's key argument; default/no-action as an outcome is missing."]}, "evalUse": "Contentious governance decision spanning three meetings with a pending vote, external authority and a recorded dissent.", "fidelity": {"method": "git-file", "truncated": true}, "notes": "Minutes of 2026-04-01 end mid-discussion with an ellipsis, and the GitHub issue (#1831) and PR (nodejs/node#62105) discussions were not read, so truncated is true. Names appear only as handles or first names in the minutes; 'Robin' in the minutes is not in the attendee list. participants.count 19 = attendees listed on 2026-04-01. The vote result is not in the repo's votes folder, which stops at 2025-08-01. A passing remark about leaked source code was omitted as not material to the decision."} +{"id": "eng-nodejs-tsc-2026-ncrypto-source-of-truth", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/nodejs/TSC/blob/main/meetings/2026-02-11.md", "kind": "minutes", "retrievedAt": "2026-10-18T05:45:00Z"}, {"url": "https://github.com/nodejs/TSC/blob/main/meetings/2026-03-18.md", "kind": "minutes", "retrievedAt": "2026-10-18T05:45:00Z"}, {"url": "https://github.com/nodejs/TSC/blob/main/meetings/2026-05-06.md", "kind": "minutes", "retrievedAt": "2026-10-18T05:45:00Z"}], "repo": "nodejs/TSC", "domain": "architecture", "title": "Source of truth for the shared crypto helper library", "question": "Should the crypto helper library's separate repository be the upstream source of truth that the main repository syncs from, or should the main repository stay authoritative for core crypto and TLS code?", "questionExplicit": false, "context": "A pull request to add an updater script for the library kept appearing on the committee agenda. The real question became where changes originate, how security releases are coordinated, and whether the separate repository tests all supported configurations.", "options": [{"label": "Separate repo is source of truth; main repo pulls via an updater script", "proposedBy": "unrecorded", "fate": "partly"}, {"label": "Main repo stays authoritative; push changes out to the separate repo", "proposedBy": "@panva", "fate": "partly"}, {"label": "Keep it in core to avoid coordination during security releases", "proposedBy": "@marco-ippolito", "fate": "deferred"}], "criteria": ["test coverage across configurations (dynamic OpenSSL versions)", "security release coordination", "contributor friction", "simplicity of sync direction"], "participants": {"count": null, "roles": ["voting members", "maintainer of the library"], "decider": "unclear"}, "process": {"research": "none", "spike": "implementation-pr", "poll": false, "escalation": "tsc", "subQuestions": ["whether BoringSSL is supported", "whether to bring beneficiaries into a call"], "deferred": true, "reopened": false, "superseded": false, "durationDays": 84, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["first meeting skipped part of the item because two relevant people were absent", "a member suggested getting the people who benefit into the call", "by May tooling existed to sync in either direction and one member said he strongly opposes routing core crypto and TLS through the separate repo"]}, "outcome": {"decided": false, "decision": "No formal decision recorded. By 2026-05-06 syncing from main to the separate repo works and the reverse is expected; the authority question stays open with one strongly stated position.", "how": "unclear", "rationale": null, "dissent": "One member said he is strongly opposed to core crypto/TLS changes going through the separate repo rather than the main repo.", "confidence": "none", "reversibility": "unknown", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "A source-of-truth choice among two flows, with facts needed on tests and release coordination.", "secondaryShape": "check", "fitsFlows": ["ask-teammate", "revisit-reopen"], "gaps": ["off-platform decision: Decision carried by an agenda-labelled PR over three meetings with different attendees; items skipped when key people are absent. The design has waiting faces for people asked, not 'needs these people in the room' or automatic carry-over.", "reframing: The question changed shape (merge an updater script, then where truth lives, then whether core crypto goes through the other repo) with no record of the reframes. F8 covers Chopin reshaping at Ada's request, not drift in what is being asked.", "implicit decision: Resolved by implementation drift: tooling for both directions arrived and became the answer by default. A decision settled by what got built.", "external authority: Stakeholders outside the voting group (consumers of the library) are named as needed but have no seat.", "blocking dependency: Factual premises nobody had (does the separate repo test every configuration; is BoringSSL supported) are check-type claims that block the choice; the design has Check as separate cards, not as prerequisites of a choice."]}, "evalUse": "Slow committee decision resolved by drift, with attendance gaps and a reframed question.", "fidelity": {"method": "git-file", "truncated": true}, "notes": "Based only on three sets of minutes; the PR (nodejs/node#61613) discussion was not read, hence truncated true. Minutes record brief fragments by first name; positions were attributed to the handles listed in attendance where the first name maps unambiguously (Filip to @panva, Marco to @marco-ippolito). Antoine's view that both approaches are sound is reported second-hand by another member in the minutes."} +{"id": "eng-nodejs-tsc-2026-macos-x64-tier2", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/nodejs/TSC/blob/main/meetings/2026-05-06.md", "kind": "minutes", "retrievedAt": "2026-10-18T05:45:00Z"}], "repo": "nodejs/TSC", "domain": "compatibility", "title": "When to downgrade macOS x64 to Tier 2 support", "question": "When should macOS x64 be moved to Tier 2 support: in a v26 minor release now, from 27.x, or only when Apple actually ends support?", "questionExplicit": false, "context": "A documentation PR proposed downgrading the platform because Apple will end x64 support before the current release line ends. No firm Apple timeline exists.", "options": [{"label": "Ship the downgrade in v26.1.0", "proposedBy": "@aduh95", "fate": "deferred"}, {"label": "Apply it from 27.x, avoiding mid-cycle changes", "proposedBy": "@RafaelGSS", "fate": "deferred"}, {"label": "Wait until Apple actually removes support", "proposedBy": "@RafaelGSS", "fate": "deferred"}], "criteria": ["setting accurate expectations", "avoiding support changes mid-release-line", "unknown Apple timeline", "build team's view"], "participants": {"count": 9, "roles": ["voting members"], "decider": "unclear"}, "process": {"research": "none", "spike": "none", "poll": false, "escalation": "tsc", "subQuestions": ["the tier change itself is treated as inevitable; the timing is disputed"], "deferred": true, "reopened": false, "superseded": false, "durationDays": null, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["one member recommended Tier 2; another called the debate academic since it will happen anyway", "build team had no representative present", "moved to an email to the rest of the committee"]}, "outcome": {"decided": false, "decision": "Not decided in the meeting; the proposer was to email the rest of the committee.", "how": "unclear", "rationale": null, "dissent": null, "confidence": "none", "reversibility": "unknown", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "Timing options for an agreed-in-principle change.", "secondaryShape": "tune", "fitsFlows": ["ask-teammate", "settle"], "gaps": ["nested follow-ups: The 'what' is accepted and only the 'when' is disputed: a parent decision agreed and a child question on timing. The design splits no agreed part from a disputed part.", "off-platform decision: The key stakeholder (build team) was absent; the discussion moved to email. There is no roster of who must weigh in, and the design assumes one chat.", "conditional decision: Timing depends on an external event with no date (Apple's end of support): a decision conditional on a trigger. 'Revisit when X is announced' has no representation.", "convention scope: Support commitments span release lines already shipped (24, 26) and future (27). A scope field, 'applies to these versions', is missing from the record.", "off-platform decision: Channel changes mid-decision (meeting, then email, then GitHub) and the record would only see the Chat."]}, "evalUse": "Short meeting item with timing options, an absent stakeholder and a move to async.", "fidelity": {"method": "git-file", "truncated": true}, "notes": "One meeting's minutes only; the PR (nodejs/node#63055) discussion was not read, so truncated is true. participants.count 9 = voting members present on 2026-05-06. Fates are 'deferred' because the minutes close without a decision."} +{"id": "python-2018-12-governance-model-vote-pep-8016", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://peps.python.org/pep-8001/", "kind": "proposal", "retrievedAt": "2026-10-18T00:00:00Z"}, {"url": "https://peps.python.org/pep-8016/", "kind": "proposal", "retrievedAt": "2026-10-18T00:00:00Z"}, {"url": "https://peps.python.org/pep-8010/", "kind": "proposal", "retrievedAt": "2026-10-18T00:00:00Z"}], "repo": "python/peps", "domain": "process-governance", "title": "Choosing CPython's post-BDFL governance model", "question": "Which governance model should the CPython project adopt now that Guido has retired as BDFL: one of seven candidate PEPs, or continued discussion?", "questionExplicit": true, "context": "Guido's retirement left the project needing to choose how it would choose a governance structure. A working group at a core sprint drafted a voting process, and seven governance PEPs in the 8010 series competed, with the goal of resolving it before Python 3.8 and certainly before PyCon US 2019.", "options": [{"label": "PEP 8016: five-person steering council elected by the core team", "proposedBy": "Nathaniel J. Smith, Donald Stufft", "fate": "chosen"}, {"label": "PEP 8010: technical leader model", "proposedBy": "Warsaw", "fate": "rejected"}, {"label": "PEP 8011: three-person leadership trio", "proposedBy": "Mariatta, Warsaw", "fate": "rejected"}, {"label": "PEP 8012: community governance model", "proposedBy": "Langa", "fate": "rejected"}, {"label": "PEP 8013: external council model", "proposedBy": "Dower", "fate": "rejected"}, {"label": "PEP 8014: commons governance model", "proposedBy": "Jansen", "fate": "rejected"}, {"label": "PEP 8015: organisation of the Python community", "proposedBy": "Stinner", "fate": "rejected"}, {"label": "Further discussion (make no choice yet)", "proposedBy": "voting process design", "fate": "rejected"}], "criteria": ["use of mature, previously tested governance practice", "simplicity", "comprehensiveness of the rules", "flexibility to change later", "timeliness", "legitimacy and secrecy of the vote"], "participants": {"count": null, "roles": ["PEP authors (core developers)", "core developers as voters", "PSF infrastructure director supervising the vote"], "decider": "vote"}, "process": {"research": "links", "spike": "none", "poll": true, "escalation": "vote", "subQuestions": ["voting method: Condorcet versus approval voting", "public versus private ballots", "whether incomplete rankings count", "who should abstain", "how ties and cycles are handled"], "deferred": false, "reopened": false, "superseded": false, "durationDays": 115, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["the voting method was itself decided first through discussion and an approval poll among core developers", "a review window preceded a two-week vote, and candidate PEPs were frozen during voting", "a 'further discussion' choice was ranked alongside real options so voters could say none was good enough", "private ballots won a poll after objections to public ones, with anonymised ballots published afterward", "inactive core developers who intended to stay inactive were asked to abstain, without enforcement"]}, "outcome": {"decided": true, "decision": "PEP 8016, the steering council model, won the core developer vote announced on 2018-12-17, and the other six models were rejected by the same vote.", "how": "vote", "rationale": "For the winning model: be boring, simple, comprehensive and flexible, borrowing mature practice from another open source project. Reasons voters ranked it first are not recorded because ballots were anonymous.", "dissent": null, "confidence": "stated", "reversibility": "stated-hard", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "A ranked pick among eight named alternatives.", "secondaryShape": null, "fitsFlows": ["settle"], "gaps": ["ranked choice: A formal ranked vote among full competing documents (Condorcet over seven candidates plus a 'none yet' sentinel), run by an outside trusted party. The nearest primitive is a stance pill on a take, with no ranking, eligibility roll or quorum.", "formal vote: Secret ballot was a stated requirement, chosen by an earlier poll. The design shows faces, names and reasons on Takes, so it cannot hold anonymous votes or publish anonymised ballots afterward.", "nested follow-ups: A meta-decision (how to decide how to decide) with its own PEP, resolved before the object-level question. Nested questions with an ordering constraint are not modelled.", "formal vote: Options are independent documents with their own authors and change histories, frozen while the vote runs. There is no option owner and no freeze or lock of options during a vote.", "rejected options content: Losers are first-class records that point to the winner (status Rejected, see winner). 'What we considered' is only a list of names.", "staged acceptance: Time-boxed phases (review window, then vote window, results on a fixed day) have no counterpart; the design has no deadline or phase for a question."]}, "evalUse": "Process decision with a formal ranked vote, eight options and a nested voting-method decision, all documented in repo text.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "PEP 8001 read in full, PEP 8016 read in part (acceptance note, rationale, specification headings and amendment rule), and PEP 8010's rejection note, from clone (python/peps clone at commit 6068058). The results announcement is linked from the PEPs on a discussion forum that could not be fetched, so vote counts and ballot detail are not recorded. Statuses of 8011 to 8015 were checked from their headers. durationDays runs from PEP 8001's creation date (2018-08-24) to the 2018-12-17 result. The seven PEP authors in options come from the candidate list inside PEP 8001. Original decider wording (v0): vote of the CPython core developers by ranked ballot (Condorcet method)."} +{"id": "python-2018-07-pep-572-assignment-expressions", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://peps.python.org/pep-0572/", "kind": "proposal", "retrievedAt": "2026-10-18T00:00:00Z"}], "repo": "python/peps", "domain": "language-design", "title": "Assignment expressions (the walrus operator)", "question": "Should Python get an expression form of assignment, and if so with what syntax, precedence and scoping rules?", "questionExplicit": true, "context": "Naming a value is only possible in statements, so comprehensions and conditions cannot capture sub-results without refactoring. Similar proposals had come up repeatedly on the ideas list. The PEP lists many alternative spellings and scoping variants that were weighed and rejected.", "options": [{"label": "NAME := expr as an operator that binds like ordinary assignment", "proposedBy": "PEP authors (Angelico, Peters, van Rossum)", "fate": "chosen"}, {"label": "expr as NAME", "proposedBy": null, "fate": "rejected"}, {"label": "expr -> NAME", "proposedBy": null, "fate": "rejected"}, {"label": "Statement-local names marked with a leading dot", "proposedBy": null, "fate": "rejected"}, {"label": "A where: block attached to a statement", "proposedBy": "prior PEP 3150", "fate": "rejected"}, {"label": "TARGET from expr", "proposedBy": null, "fate": "rejected"}, {"label": "Capture only in if and while statements", "proposedBy": null, "fate": "rejected"}, {"label": "Statement-scoped (sublocal) names to avoid namespace pollution", "proposedBy": "earlier revisions of this PEP", "fate": "rejected"}, {"label": "Lower precedence so most conditions need no parentheses", "proposedBy": null, "fate": "rejected"}, {"label": "Allow an unparenthesised tuple on the right-hand side", "proposedBy": "critics", "fate": "rejected"}, {"label": "Always require parentheses", "proposedBy": null, "fate": "rejected"}, {"label": "Change comprehension scope rules as part of this PEP", "proposedBy": "earlier revision of this PEP", "fate": "rejected"}], "criteria": ["no confusion with existing as syntax", "no new keyword", "consistency with the assignment statement", "simple scope rules", "usefulness in comprehensions and conditions", "readability on real code"], "participants": {"count": null, "roles": ["three co-authors including the BDFL", "ideas-list and developer-list participants", "core developers"], "decider": "BDFL"}, "process": {"research": "data-analysis", "spike": "prototype", "poll": false, "escalation": "none", "subQuestions": ["precedence of the operator", "scope of the target inside comprehensions and class bodies", "evaluation order in dictionary comprehensions", "when parentheses are required"], "deferred": false, "reopened": false, "superseded": false, "durationDays": null, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["the PEP header lists nine posting dates between February 2018 and August 2019", "a scope-rule change for comprehensions was withdrawn so the PEP could stay focused", "the sublocal-scope idea from earlier revisions was dropped for simplicity", "the PEP argues for judging by real code, with appendix findings from reading standard-library code, rather than toy examples", "after acceptance and before release the spec was amended: a dedicated exception subclass was dropped and use in comprehension iterables was restricted"]}, "outcome": {"decided": true, "decision": "Accepted by the BDFL in July 2018 (the PEP header links the resolution message) and shipped in Python 3.8, with spec amendments made before release.", "how": "maintainer-decision", "rationale": "Naming sub-expressions inline helps comprehensions, conditions and debugging, and the chosen operator avoids the conflicts and awkwardness of the alternative spellings while keeping scope rules simple.", "dissent": "The PEP records recurring objections (why not make = an expression, why keep statements, namespace pollution) and answers them; remaining individual dissent is not recorded in it.", "confidence": "implied", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "A syntax and semantics choice among many named alternatives, plus coupled scope and precedence sub-choices.", "secondaryShape": "check", "fitsFlows": ["research-or-spike", "settle"], "gaps": ["delegated decider: The decider (BDFL) is also a co-author. The design has no conflict-of-interest marker, and the closest analogue is the asker pressing Settle, which conflates champion and decider.", "off-platform decision: Decision recorded as a pointer to a mailing-list message after nine rounds of a living document. 'What we knew at the time' would need snapshots of each revision because the PEP is edited in place.", "rejected options content: Rejected ideas are a dozen separate entries, each with its own mini-argument and code example. The source treats them as first-class content, while 'What we considered' lists option names only.", "coupled decisions: Coupled sub-decisions (spelling, precedence, target scope, parentheses, comprehension scope, evaluation order) were accepted together. The model must either make each a question or fold them into one card.", "evidence beyond check: Evidence was real-code reading (standard-library examples and an appendix essay) with a stated norm against toy examples. Research modes in the design do not include 'examples from real code'.", "superseded or moved to another thread: The spec was amended between acceptance and release. F6 Reopen implies a full reopen, not a small patch recorded on the settled answer."]}, "evalUse": "Well-known language decision with a dozen explicit rejected alternatives and an author who is also the decider.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "PEP read in the relevant sections from clone (python/peps clone at commit 6068058): header, rationale, spec-changes, rejected alternatives, objections and appendix start. The resolution link is a mailing-list message that was not retrievable here, so the acceptance wording and exact date are not read. durationDays is null because the header gives no exact resolution date; participant and comment counts are not in the PEP. Proposers are null where the PEP does not attribute an alternative. Original decider wording (v0): BDFL (also a co-author)."} +{"id": "python-2023-10-pep-703-optional-gil", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://peps.python.org/pep-0703/", "kind": "proposal", "retrievedAt": "2026-10-18T00:00:00Z"}], "repo": "python/peps", "domain": "architecture", "title": "Optional global interpreter lock in CPython", "question": "Should CPython accept a build mode that runs without the global interpreter lock, given single-thread overhead, an ABI split and the cost to the extension ecosystem?", "questionExplicit": true, "context": "The lock limits multi-core use, which matters for scientific and AI workloads. The author, an outside contributor with a core-developer sponsor, proposed a build flag with biased reference counting, per-object locks and a new allocator, and measured overhead on a standard benchmark suite.", "options": [{"label": "Accept the PEP: a --disable-gil build mode, with a later path toward one build", "proposedBy": "Sam Gross (author), sponsor Lukasz Langa", "fate": "chosen"}, {"label": "Keep the lock and use multiprocessing for parallelism", "proposedBy": null, "fate": "rejected"}, {"label": "Keep the lock and release it inside C extensions", "proposedBy": null, "fate": "rejected"}, {"label": "Parallelise inside native libraries", "proposedBy": null, "fate": "rejected"}, {"label": "Use a concurrent garbage collector", "proposedBy": null, "fate": "rejected"}, {"label": "Merge the two builds into one runtime-switchable lock in later phases", "proposedBy": "PEP open-issues section", "fate": "deferred"}], "criteria": ["multi-core use for scientific and AI workloads", "single-thread overhead (about 5 to 6 percent reported)", "ABI and extension compatibility", "maintenance burden", "ability to roll back"], "participants": {"count": null, "roles": ["author", "sponsor core developer", "steering council", "extension and scientific-library maintainers"], "decider": "vote"}, "process": {"research": "benchmarks", "spike": "prototype", "poll": false, "escalation": "steering", "subQuestions": ["a three-phase path to a single build mode through about 2030", "specialisation under free threading", "mitigating single-thread slowdown", "handling of borrowed references in the C API"], "deferred": false, "reopened": false, "superseded": false, "durationDays": 288, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["a second discussion thread was opened in May 2023, per the PEP's post history", "the council's acceptance is a note on the PEP with conditions, announced on a discussion forum", "the council reserved the right to roll back the whole change if it proves too disruptive"]}, "outcome": {"decided": true, "decision": "Accepted by the steering council on 2023-10-24 on condition that rollout is gradual, breaks as little as possible, and can be rolled back, including reverting the entire PEP.", "how": "unclear", "rationale": "Not in the PEP text; only the conditions are noted. The acceptance announcement is on a discussion forum that could not be fetched.", "dissent": null, "confidence": "stated", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "Accept or decline against alternatives, with a measured cost to weigh.", "secondaryShape": "check", "fitsFlows": ["research-or-spike", "settle", "build-pr"], "gaps": ["conditional decision: Conditional, staged acceptance by a body: first phase accepted with provisos and a standing right to roll back everything. The design has Settled plus Reopen and no 'accepted with monitored conditions' state or condition tracker.", "delegated decider: A council decides over the author, and neither the author nor the sponsor can settle. There is no flow where the asker hands the question to a decision body and the result comes back from another platform.", "staged acceptance: A multi-year roadmap sits in the doc (three phases to around 2030) where only phase one is decided. Later phases are open issues to revisit on a schedule or condition, not on a person pressing Reopen.", "conditional decision: The trade-off turns on benchmark numbers (about 5 to 8 percent overhead) and ecosystem readiness that keep moving after acceptance. 'Decision conditional on a measured threshold' has no home in the design.", "votes as evidence: Key stakeholders are ecosystem classes (scientific and AI library maintainers, extension authors), represented as motivation sections. Takes models individual teammates.", "rejected options content: Rejected ideas and open issues are explicit sections of the source, i.e. first-class content on the record."]}, "evalUse": "Council-decided, conditionally accepted proposal with a multi-phase roadmap and explicit performance numbers.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "PEP read in the relevant sections from clone (python/peps clone at commit 6068058): header and acceptance note, performance, alternatives, rejected ideas and open issues. The acceptance post and discussion threads linked from the header could not be fetched, so rationale, vote split and dissent are unknown. durationDays runs from creation (2023-01-09) to the resolution date (2023-10-24). Whether the council voted unanimously is not recorded in the PEP. Original decider wording (v0): steering council (elected five-member body), decision by council vote."} +{"id": "python-2023-05-pep-649-deferred-annotations-supersedes-563", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://peps.python.org/pep-0563/", "kind": "proposal", "retrievedAt": "2026-10-18T00:00:00Z"}, {"url": "https://peps.python.org/pep-0649/", "kind": "proposal", "retrievedAt": "2026-10-18T00:00:00Z"}, {"url": "https://peps.python.org/pep-0749/", "kind": "proposal", "retrievedAt": "2026-10-18T00:00:00Z"}], "repo": "python/peps", "domain": "language-design", "title": "How annotations are evaluated: stringified versus deferred", "question": "How should Python evaluate annotations: make the stringified postponed form from PEP 563 the default, or replace it with lazily evaluated annotations as in PEP 649?", "questionExplicit": true, "context": "PEP 563 added a future import that stores annotations as strings, with the idea that it might become the default. Runtime users of annotations were hurt by the change. PEP 649 proposed lazy evaluation through a generated function, was blocked for years on satisfying every use case, and was accepted in 2023 for a later release; PEP 749 later fixed details.", "options": [{"label": "PEP 649: evaluate annotations lazily via a generated annotate function", "proposedBy": "Larry Hastings", "fate": "chosen"}, {"label": "PEP 563: store annotations as strings, with the future import becoming the default", "proposedBy": "Lukasz Langa", "fate": "rejected"}, {"label": "A lazily evaluated mapping built over stored annotation text", "proposedBy": "Yury Selivanov", "fate": "rejected"}, {"label": "Keep eager evaluation at definition time (stock semantics)", "proposedBy": null, "fate": "rejected"}, {"label": "Store the original source text for a source format", "proposedBy": "participants in the PEP 649 discussion", "fate": "deferred"}], "criteria": ["forward references", "runtime introspection needs", "static typing users", "backward compatibility", "performance and import cost", "access to local names"], "participants": {"count": null, "roles": ["PEP authors", "typing and runtime-annotation library maintainers", "core developers", "steering council"], "decider": "unclear"}, "process": {"research": "links", "spike": "prototype", "poll": true, "escalation": "none", "subQuestions": ["which annotation formats to support", "the retirement timeline for the future import", "the module that exposes annotation tooling", "REPL behaviour"], "deferred": true, "reopened": false, "superseded": true, "durationDays": 847, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["PEP 649 was blocked until a contributor proposed a technique that satisfied all runtime and typing use cases", "the discussion thread was retitled to note tentative acceptance in November 2022, with formal resolution in May 2023", "two polls on how to revise the PEP and a major rewrite with course corrections came before the final text", "the planned release moved from 3.10 to 3.14, and PEP 563's default status never happened", "PEP 749 later amended 649 and kept the future import until a stated end-of-life, then deprecation"]}, "outcome": {"decided": true, "decision": "PEP 649 was resolved on 2023-05-08 and marked final for Python 3.14; PEP 563 is superseded and never became default; PEP 749 (resolved 2025-05-05) set the details and the future import's retirement path.", "how": "unclear", "rationale": "Stated in PEP 649: lazy evaluation fixes forward references without stringifying, keeps annotations usable at runtime, and is fast, while PEP 563 hurt runtime users and could not see local names.", "dissent": null, "confidence": "stated", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "Two competing mechanisms and several rejected variants.", "secondaryShape": "check", "fitsFlows": ["research-or-spike", "settle", "revisit-reopen"], "gaps": ["staged acceptance: Tentative acceptance followed by revisions and a formal acceptance months later: a staged decision with states 'tentatively accepted', 'revised after polls' and 'accepted'. The design has Open and Settled only.", "superseded or moved to another thread: A chain of three records (563, then 649, then 749) where the earlier decision's planned future (becoming the default) was cancelled. There is no supersedes link between records, only Reopen on one.", "blocking dependency: Blocked for years on a precondition: it could not be accepted until it satisfied every runtime and typing use case, and the unlocking technique came from another contributor. 'Blocked on a technical artefact' has no status.", "formal vote: Polls were about how to revise the proposal, not about the final verdict. Meta-polls on option shape are not in the design.", "staged acceptance: The decision date and the shipping release diverge by several versions. Built into shows PRs, not a planned-versus-actual release slip.", "deferred: 'Just store the strings' was labelled not yet instead of rejected. An option can be deferred with a revisit hint inside a settled question, which the Also considered list cannot express."]}, "evalUse": "Multi-year supersession chain with tentative acceptance, polls on revisions, and a deferred option inside an accepted PEP.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "PEP 563 header, resolution and deprecation sections, PEP 649 header, rejected ideas and backward-compatibility sections, and PEP 749 abstract and background were read from clone (python/peps clone at commit 6068058). Threads for acceptance, polls and the rewrite are linked in post history but could not be fetched. The decider is not named in the PEP text, so it is left unclear even though a council decision is plausible. durationDays runs from PEP 649's creation (2021-01-11) to its resolution date (2023-05-08). The tentative-acceptance wording is inferred from a discussion thread title in the header. Original decider wording (v0): unclear (the PEP header links a resolution but does not name the decider)."} +{"id": "python-2019-06-pep-602-annual-release-cycle", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://peps.python.org/pep-0602/", "kind": "proposal", "retrievedAt": "2026-10-18T00:00:00Z"}], "repo": "python/peps", "domain": "process-governance", "title": "Annual release cycle for Python", "question": "Should CPython move from an eighteen-month feature release cycle to a predictable twelve-month cycle with a fixed October release?", "questionExplicit": true, "context": "The release manager proposed shortening the cycle, keeping five years of support per version, and aligning with distributions. The PEP compares the new calendar against the old one and lists dependent policies that stay unchanged.", "options": [{"label": "Twelve-month cycle with seventeen months of development per version and five years of support", "proposedBy": "Lukasz Langa (author, release manager)", "fate": "chosen"}, {"label": "Keep the eighteen-month cadence", "proposedBy": null, "fate": "rejected"}, {"label": "Nine-month cadence", "proposedBy": "PEP 596", "fate": "rejected"}, {"label": "Keep four betas over four months and a one-month release candidate", "proposedBy": null, "fate": "rejected"}, {"label": "Slow releases without freezing features at the first beta (incremental feature releases)", "proposedBy": "PEP 598", "fate": "rejected"}, {"label": "Designated long-term-support releases", "proposedBy": null, "fate": "rejected"}], "criteria": ["predictable calendar", "smaller and less risky releases", "synchronisation with distributions such as Fedora", "growth of the testing matrix", "core developer scheduling", "unchanged support length"], "participants": {"count": null, "roles": ["author who is the active release manager", "PEP delegate", "core developers", "distribution maintainers"], "decider": "named maintainer"}, "process": {"research": "links", "spike": "none", "poll": false, "escalation": "none", "subQuestions": ["whether to switch to calendar versioning (deferred to another PEP)", "the deprecation policy under a faster cadence", "the council's term length", "the release manager's term length"], "deferred": false, "reopened": false, "superseded": false, "durationDays": null, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["the author disclosed that the change shortens his own term as release manager and offered to cover an extra release", "dependent policies were listed explicitly, mostly as kept unchanged", "the versioning-scheme question was cut out as a non-goal", "the doc was later updated to reflect a longer full-support period starting with a later version"]}, "outcome": {"decided": true, "decision": "Accepted: annual October releases beginning with Python 3.9. The PEP is now a living Process document.", "how": "maintainer-decision", "rationale": "Stated in the PEP: releases get smaller and less risky, features reach users sooner, the calendar becomes predictable, there is less pressure to rush before the first beta, and distributors can sync with their own schedules.", "dissent": null, "confidence": "stated", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "A cadence picked among named alternatives with a dated calendar comparison.", "secondaryShape": "tune", "fitsFlows": ["settle", "revisit-reopen"], "gaps": ["delegated decider: A delegate appointed for this one question (PEP-Delegate in the header) decides, not the author and not a standing body. Per-question decider assignment recorded in the document is not modelled.", "delegated decider: The author's conflict of interest is explicit in the text, with compensation offered. The design has no way to flag or disclose a conflict.", "visual comparison: The options include a numeric dial (months between releases: 9, 12, 18) with consequences shown as dated calendars and a support matrix. The tune shape previews a visual but not a dated schedule.", "nested follow-ups: A follow-on question (calendar versioning) was explicitly split off as a non-goal for another PEP. Settling a question that spawns a named future question is not a flow.", "consequences and obligations: Dependent policies are enumerated as 'kept as is'. Consequences that stay unchanged are part of the answer but have no slot on the record.", "superseded or moved to another thread: The PEP is edited in place after acceptance (support length changed under a later PEP). Amending a settled process document without reopening it is not modelled."]}, "evalUse": "Process PEP with a delegate decider, explicit author conflict disclosure and a calendar comparison.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "PEP read in full from clone (python/peps clone at commit 6068058). The resolution message is a mailing-list link that could not be fetched, so the acceptance date and wording are not read, and durationDays is null. The 596 and 598 rejections are described as the PEP describes them. Participant count and comment counts are not in the PEP. Original decider wording (v0): named maintainer (the PEP-Delegate in the header)."} +{"id": "python-2007-pep-3103-switch-statement-rejected", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://peps.python.org/pep-3103/", "kind": "proposal", "retrievedAt": "2026-10-18T00:00:00Z"}], "repo": "python/peps", "domain": "language-design", "title": "A switch/case statement for Python", "question": "Should Python add a switch/case statement, and with which syntax and dispatch semantics?", "questionExplicit": true, "context": "After a flurry of discussion on the developer list, the BDFL wrote a PEP to pull his own preferences out of the many proposals. It names the syntax variants, schools of semantics and freezing options so the discussion can refer to them.", "options": [{"label": "A switch with cases indented under it (alternative 1)", "proposedBy": "PEP 275 preference; BDFL converted to it", "fate": "rejected"}, {"label": "A switch with unindented cases (alternative 2)", "proposedBy": "Fredrik Lundh", "fate": "rejected"}, {"label": "Restrict case expressions to literals frozen at compile time", "proposedBy": "Raymond Hettinger", "fate": "rejected"}, {"label": "Wait for a proposal for pre-computed case values before deciding", "proposedBy": "BDFL (PEP conclusion)", "fate": "deferred"}, {"label": "No switch; keep if/elif chains", "proposedBy": null, "fate": "chosen"}], "criteria": ["less repetition than if/elif chains", "efficiency", "avoiding fallthrough mistakes", "fit with dynamic semantics where constants are ordinary variables", "added complexity versus mild gains"], "participants": {"count": null, "roles": ["author who is the BDFL", "developer-list participants", "conference audience"], "decider": "BDFL"}, "process": {"research": "links", "spike": "none", "poll": true, "escalation": "none", "subQuestions": ["syntax layout", "syntax for grouping several values per case", "if/elif semantics versus dict-based dispatch", "when the dispatch table is frozen"], "deferred": true, "reopened": false, "superseded": false, "durationDays": null, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["the author gave names to variants such as alternative numbers and schools so participants could refer to them", "the PEP's conclusion said it was too early to decide and asked for a pre-computed-values proposal first", "the rejection notice cites a quick audience poll during a conference keynote showing little support"]}, "outcome": {"decided": true, "decision": "Rejected by the BDFL after a show-of-support poll at a conference keynote; Python stayed without a switch statement.", "how": "maintainer-decision", "rationale": "Per the rejection notice, the audience poll showed no popular support. The PEP text itself found the gains mild and wanted more evidence before deciding.", "dissent": null, "confidence": "stated", "reversibility": "unknown", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "A feature compared across syntax variants, semantic schools and a no-change option.", "secondaryShape": null, "fitsFlows": ["research-or-spike", "settle"], "gaps": ["formal vote: Decision by a live show-of-hands poll at a conference keynote. The poll was off-platform, uncounted and summarised by the decider. The design has no poll primitive and no way to record an offline poll.", "delegated decider: The author, BDFL and decider are the same person, who writes the options, ranks them and rejects the whole thing. The PEP is both the research document and the verdict.", "coupled decisions: The option space is a matrix of independent axes (layout, case grouping, semantic school, freezing time) with names like alternative 1 and school II. The choice shape is a flat list.", "blocking dependency: The interim position was 'too early; wait for a specific proposal', i.e. a precondition expressed as a missing artefact, followed by a rejection without that artefact arriving. Blocked-on-artefact is not a status.", "rejected options content: The rejected PEP is kept as the reference record years later. The whole document is the record of rejected ideas, and the design's record page is a derived summary instead."]}, "evalUse": "Rejection by an audience poll, with a rich option matrix preserved in the document.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "PEP read in the rejection notice, abstract, rationale, syntax alternatives, discussion, semantics freeze section and conclusion from clone (python/peps clone at commit 6068058). The date of the keynote poll is not given in the PEP, so durationDays is null. The attributions to named people are as stated in the PEP text. Reversibility is unknown because the PEP says nothing about revisiting."} +{"id": "tc39-2019-12-dynamic-code-brand-checks-stage2-refused", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/tc39/notes/blob/main/meetings/2019-07/july-25.md#dynamic-code-brand-checks-for-stage-2", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}, {"url": "https://github.com/tc39/notes/blob/main/meetings/2019-12/december-5.md#dynamic-code-brand-checks-for-stage-2", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}], "repo": "tc39/notes", "domain": "security", "title": "Advance Dynamic Code Brand Checks to Stage 2 (refused twice)", "question": "Should the dynamic code brand checks proposal, which lets hosts restrict eval-style code generation to values carrying a code-like brand, be accepted into Stage 2 while its brand mechanism and host hooks are still contested?", "questionExplicit": true, "context": "MSL (a Google security engineer) proposed a brand check plus host hooks so that eval and Function-style sinks only accept values marked as trusted code, motivated by Trusted Types on the web. It was requested for Stage 2 in July 2019 and again in December 2019 after revisions.", "options": [{"label": "Grant Stage 2 now and fix the host hooks before Stage 3", "proposedBy": "YK, MSL", "fate": "rejected"}, {"label": "Hold at Stage 1 until host hooks can tell direct from indirect eval and strict from sloppy code, and the brand mechanism is firmer", "proposedBy": "MM, WH", "fate": "chosen"}, {"label": "Brand mechanism: internal slot versus symbol versus something else", "proposedBy": "MM (objected to slot and symbol), WH (uneasy about attack surface)", "fate": "deferred"}, {"label": "Let the objector review the host hooks over lunch and then grant Stage 2 if satisfied", "proposedBy": "MM", "fate": "partly"}], "criteria": ["attack surface of the brand check (can brands be forged)", "host hooks must be virtualizable and distinguish eval flavours", "whether a gap is a Stage 2 entrance blocker or a Stage 3 requirement", "late changes inside the 10-day agenda window", "maturity of the Trusted Types work in W3C", "new verb for eval-like operations"], "participants": {"count": 11, "roles": ["champion (Google security)", "security-minded reviewer volunteers", "implementers (V8, Mozilla)", "Node.js delegate", "spec editor", "framework author"], "decider": "consensus"}, "process": {"research": "links", "spike": "none", "poll": false, "escalation": "none", "subQuestions": ["Must host hooks carry direct/indirect and strict/sloppy information before Stage 2?", "How is forging of brands prevented?", "Are dynamic import and module-map poisoning code-generation vectors that need hooks?", "Do host hooks count as API for entrance-criteria purposes?"], "deferred": true, "reopened": true, "superseded": false, "durationDays": 133, "comments": 60, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["July: two delegates offered to review only if it ever reached Stage 2, which the champion took as a refusal", "December: the entrance criteria were read out mid-debate to decide whether the objection was a blocker", "December: objector reviewed the host hooks during the lunch break and reported they were still inadequate", "A delegate objected that a major semantic change landed inside the 10-day pre-meeting window"]}, "outcome": {"decided": false, "decision": "Stage 2 was not granted at either meeting; the champion was to work with the main objector and others before the next meeting.", "how": "default/no-action", "rationale": "Host hooks could not yet distinguish eval variants, the brand mechanism was vague and risky, and a significant change arrived late.", "dissent": "MM would accept Stage 2 only once the hooks carry the missing information; WH wanted the brand mechanism pinned down before Stage 3 at the latest; KG disliked adding a new verb for eval-like operations.", "confidence": "stated", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "check", "shapeWhy": "The question is a readiness verdict against stated entrance criteria, with a secondary choice about the brand mechanism.", "secondaryShape": "choice", "fitsFlows": ["ask-teammate", "settle", "revisit-reopen"], "gaps": ["conditional decision: No honest way to close with a conditional not-yet: Settle needs an answer sentence, but the recorded result is a refusal plus conditions, and the question must stay open with a history of two failed attempts five months apart.", "consensus mechanics: A single unresolved objection decides the outcome (unanimity): the Objects stance has no force, and the record needs blocked-by-MM-pending-X as distinct from simply disagrees.", "convention scope: Decision is judged against an external rubric (the process document's entrance criteria) that participants read differently; the design has criteria columns for options but no gate checklist owned by a process and arguable in the thread.", "delegated decider: Reviewer roles change positions: volunteering to review is a precondition of support, but Ask someone is a request for a take, not a formal reviewer assignment with a deliverable.", "consensus mechanics: A procedural objection about timing (10-day agenda rule) is orthogonal to the merits and has no home in Takes or Arguments.", "off-platform decision: How it moved would have to span two non-contiguous meetings with different people in the room and a revised proposal in between."]}, "evalUse": "Stage-gate refusal with a substantive security objection, a second attempt, and a live dispute over what counts as an entrance blocker; good for testing undecided or conditional outcomes.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "Both agenda items read in full from the cloned repo. The July item is short; the December item contains a lunch break after which the objector reported back. The first meeting's conclusion lists two potential reviewers; the second records that Stage 2 was refused. participants.count and process.comments are lower bounds from counting speaker tags in the transcript. Original decider wording (v0): consensus of committee (unanimity; any delegate can block)."} +{"id": "tc39-2019-10-ecma402-casing-conventions", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/tc39/notes/blob/main/meetings/2019-07/july-24.md#casing-conventions", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}, {"url": "https://github.com/tc39/notes/blob/main/meetings/2019-10/october-2.md#revisit-ecma-402-casing-conventions", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}], "repo": "tc39/notes", "domain": "naming", "title": "Casing convention for multi-word string values in ECMA-402 APIs", "question": "Should new ECMA-402 APIs spell multi-word string values and keys in camelCase, or follow the web platform and Unicode habit of kebab-case for strings?", "questionExplicit": true, "context": "SFC, working on Intl proposals including DisplayNames and a formatting proposal, found that values returned by one API are fed back as keys or inputs to another, and that web guidance (W3C TAG) favours kebab-case for strings. The plenary was asked for an opinion before the Intl group fixed its style guide.", "options": [{"label": "Use camelCase everywhere in ECMA-402 going forward (called option 3 in the slides)", "proposedBy": "SFC, LEO (ECMA-402 group)", "fate": "chosen"}, {"label": "Use kebab-case for string values, but camelCase when the string names a member of another API (called option 2 in the slides)", "proposedBy": "DD, YK, AKI", "fate": "rejected"}, {"label": "Accept both spellings and canonicalize", "proposedBy": "KM", "fate": "rejected"}, {"label": "No blanket rule; decide case by case", "proposedBy": "WH", "fate": "partly"}], "criteria": ["consistency with W3C and web-platform precedent", "consistency with Unicode locale identifiers (kebab-case)", "round-tripping an output value back in as an input", "keeping values usable as object keys later", "shipped behaviour cannot be changed", "avoiding two spellings for one thing"], "participants": {"count": 8, "roles": ["ECMA-402 editor", "Intl champion", "web framework author", "ECMAScript spec editor", "chair"], "decider": "team"}, "process": {"research": "links", "spike": "none", "poll": false, "escalation": "none", "subQuestions": ["Does the recommendation bind only ECMA-402 or all of TC39?", "Should the Intl group give more weight to the W3C precedent?", "What happens to already-shipped APIs?"], "deferred": true, "reopened": false, "superseded": false, "durationDays": 70, "comments": 67, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["July: the room declined to endorse a blanket rule; one delegate said he lacked the data to take any side", "July: a delegate asked for consensus that 402 may decide within its scope while anything wider needs the committee", "October: the Intl group returned with a style-guide PR that recorded the decision and rationale; approved without objection after a request to make the decision wording clearer", "The numbering of options came from slides that are not reproduced in the notes"]}, "outcome": {"decided": true, "decision": "July ended with no recommendation and the topic moved offline. In October the ECMA-402 style guide was approved recommending camelCase for keys as well as string values, accepting divergence from the W3C kebab-case guidance.", "how": "consensus", "rationale": "Values produced by one API are reused as inputs and may later become keys, and shipped behaviour cannot be changed without breaking the web.", "dissent": "Several delegates would have preferred web-ecosystem consistency but were content to let the Intl group decide; WH objected only to unclear decision wording.", "confidence": "implied", "reversibility": "stated-hard", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "A handful of named options compared on consistency and round-tripping criteria.", "secondaryShape": null, "fitsFlows": ["ask-teammate", "settle"], "gaps": ["delegated decider: Advice versus authority: the plenary advised but a sub-group decides; the design has one settler and no recommended-by-X, decided-by-Y split.", "convention scope: The scope of the decision (only 402, or all of TC39) was negotiated mid-thread; a question has no editable applies-to scope.", "consensus mechanics: Delegates who decline to take any side for lack of information are neither Agrees nor Objects; Unsure carries no statement of what evidence would change that.", "staged acceptance: Final settlement happened two months later by approving a style-guide PR; the earlier meeting is recorded as no recommendation, so the record needs a two-stage timeline and a Built-into item that is a document, not code.", "convention scope: A policy applies to future, not yet existing items and should not be retroactive; there is no way to say applies going forward or to track later conformance.", "reframing: The option set grew in the room (canonicalization, case by case) and the original option numbering was lost from the record."]}, "evalUse": "Policy-style naming decision that stalls, is delegated, and is finalized by a document PR in a later meeting; good for testing advice-versus-authority and scope handling.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "July item and the October revisit item both read in full. Option 1 of the slide deck is never described in the notes, so it is not listed. Reversibility is stated for already-implemented APIs only (editor said these cannot be changed). participants.count and process.comments are lower bounds from counting speaker tags in the transcript. Original decider wording (v0): ECMA-402 task group (delegated), with committee advice; wider scope would need committee consensus."} +{"id": "tc39-2019-10-proposal-disclosure-policy", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/tc39/notes/blob/main/meetings/2019-10/october-2.md#proposal-disclosure-policy", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}, {"url": "https://github.com/tc39/notes/blob/main/meetings/2019-10/october-2.md#proposal-disclosure-policy-2-electric-lawyeroo", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}], "repo": "tc39/notes", "domain": "process-governance", "title": "Disclosing who a champion is working on behalf of", "question": "Should TC39 adopt a norm or rule asking champions to disclose when they advance a proposal on behalf of, or in a contract with, another company, and where would it be written down?", "questionExplicit": true, "context": "MLS (Apple) presented slides arguing for transparency about interests behind proposals. Delegates from consultancies said client contracts and NDAs may forbid disclosure and that a requirement could create legal obligations for them.", "options": [{"label": "Mandatory disclosure policy", "proposedBy": "MLS (initial slides)", "fate": "rejected"}, {"label": "Non-binding best-practice norm, documented in the existing participation guide", "proposedBy": "MM, MLS", "fate": "partly"}, {"label": "Add champion, author and collaborator fields to the proposal template or the process document", "proposedBy": "GFC", "fate": "deferred"}, {"label": "Record it in the How We Work repository", "proposedBy": "DE", "fate": "deferred"}, {"label": "Send it to the Ecma General Assembly or ExeCom because of legal implications", "proposedBy": "WH, MBS, LEO", "fate": "deferred"}, {"label": "Put the duty on the paying party rather than the contractor", "proposedBy": "MF", "fate": "merged"}], "criteria": ["transparency about interests and bias", "legal exposure and NDAs for contractors", "burden on champions", "enforceability and tone (norm versus rule)", "avoiding harm to small consultancies", "which venue has authority"], "participants": {"count": 16, "roles": ["proposer from a browser vendor", "consultancy delegates", "Ecma secretariat staff", "chair", "spec editors", "framework author"], "decider": "consensus"}, "process": {"research": "none", "spike": "none", "poll": false, "escalation": "none", "subQuestions": ["Mandatory rule or soft norm?", "Who must disclose when two members contract with each other?", "Which document holds it: participation guide, How We Work, process document, proposal template?", "Does it need Ecma legal or General Assembly review?"], "deferred": true, "reopened": false, "superseded": false, "durationDays": 0, "comments": 76, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["Within the day the proposal was reframed from a rule to a non-binding norm after contractors raised legal worries", "WH declined to endorse anything until he had spoken to his company's lawyers", "The chair asked for resolution within a few weeks", "Two sessions on the same day: a first discussion, then a follow-up session later in the day"]}, "outcome": {"decided": false, "decision": "No conclusion. The room voiced no objection to a non-binding disclosure norm, but its wording, home document and legal review were left open; the first session explicitly recorded no conclusion.", "how": "unclear", "rationale": "Transparency about interests helps the committee interpret positions, but the group could not agree on a mechanism that avoids legal exposure for contractors.", "dissent": "WH considered consensus premature pending legal advice; MF wanted to see concrete wording; YK was not in the room for the second session.", "confidence": "stated", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "explore", "shapeWhy": "Open policy question whose option set only emerged during discussion.", "secondaryShape": "choice", "fitsFlows": ["ask-teammate", "revisit-reopen"], "gaps": ["external authority: One participant would neither agree nor object until a third party (his company's lawyers) answered; Waiting for Luke assumes the waited-on person is a teammate in the session, not an external authority.", "reframing: The question changed meaning during the thread (rule to norm); F8 handles a change of shape, but nothing versions the question text or marks when its meaning moved.", "blocking dependency: A sub-question about which body has authority (TC39 versus the Ecma General Assembly) blocks the substance, while the design assumes the asker can settle.", "consensus mechanics: Some stakeholders cannot state their reasons because of confidentiality, yet Takes expect a reason with every stance.", "off-platform decision: The eventual home of the decision was undecided between four documents, so Built into cannot name where it lands.", "deferred: The chair set a follow-up deadline of a few weeks; questions have no due date or owner for the next step."]}, "evalUse": "Policy question that stalls because of legal and venue uncertainty; good for testing no-decision outcomes and reframing mid-thread.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "Both sessions read in full; the second section ends without a Conclusion heading. Speaker attribution for a few lines in the first session is unclear in the original (marked with question marks), so the roles above are inferred from the speaker tags. participants.count and process.comments are lower bounds from counting speaker tags in the transcript. Original decider wording (v0): consensus of committee, with possible Ecma General Assembly involvement."} +{"id": "tc39-2020-04-weakrefs-finalizationregistry-cleanup", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/tc39/notes/blob/main/meetings/2020-03/april-1.md#weakrefs-finalizationregistry-api-change", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}, {"url": "https://github.com/tc39/notes/blob/main/meetings/2020-03/april-2.md#revisit-weakrefs-finalizationregistry-api-change", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}], "repo": "tc39/notes", "domain": "api-design", "title": "FinalizationRegistry callback shape and the fate of cleanupSome", "question": "Should FinalizationRegistry pass held values to the callback one at a time instead of exposing iterators, and should cleanupSome stay in the Stage 3 WeakRefs proposal given a main-thread compatibility objection?", "questionExplicit": true, "context": "WeakRefs was already at Stage 3 and the V8 champion wanted to ship soon. The champions proposed replacing the cleanup iterators (originally a Mozilla suggestion) with per-item callbacks, and a WebKit engineer objected late that cleanupSome would encourage long-running synchronous code on the web's main thread.", "options": [{"label": "Replace the cleanup iterators with per-item callbacks", "proposedBy": "DE, SYG (champions); YSV (Mozilla) supported", "fate": "chosen"}, {"label": "Keep cleanupSome as specified", "proposedBy": "MM, GCL, champions", "fate": "partly"}, {"label": "Remove cleanupSome or demote the proposal to Stage 2", "proposedBy": "floated in discussion after KM (WebKit) raised the concern", "fate": "rejected"}, {"label": "Move cleanupSome into a separate proposal", "proposedBy": "KM (open to it)", "fate": "rejected"}, {"label": "Make cleanupSome normative-optional: hosts decide at realm creation whether the method exists", "proposedBy": "KM, MM, YSV, DE", "fate": "chosen"}, {"label": "Add a per-agent flag, similar to the one for blocking waits, so hosts can vary behaviour by thread", "proposedBy": "KM", "fate": "rejected"}], "criteria": ["web compatibility risk on the main thread", "need for long-running compute loops in workers, WASM and Node", "parsimony of host hooks", "feature detectability", "not delaying shipment of WeakRefs", "layering between ECMAScript and HTML"], "participants": {"count": 10, "roles": ["champions", "implementers (V8, SpiderMonkey, JavaScriptCore)", "object-capability delegates", "Node.js delegates", "spec editor"], "decider": "consensus"}, "process": {"research": "none", "spike": "implementation-pr", "poll": false, "escalation": "none", "subQuestions": ["Wording that keeps cleanup callbacks at job boundaries", "Liveness wording and simultaneous clearing of weak refs", "Use cases that need cleanupSome", "Whether main-thread policy belongs in ECMAScript or HTML"], "deferred": false, "reopened": true, "superseded": false, "durationDays": 1, "comments": 107, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["The item split: the iterator change got consensus on day one while cleanupSome was revisited the next day", "The objecting implementer said the position came from leadership at his organization and could not be moved in the room", "MM warned that engines shipping without cleanupSome would upset him, which shaped the compromise", "The objector apologised for raising the concern late"]}, "outcome": {"decided": true, "decision": "Consensus to drop the cleanup iterators in favour of passing held values individually, and to make cleanup on FinalizationRegistry.prototype optional in the strict sense that hosts either provide the method or not at realm creation; what to do on particular threads is left to the HTML layering PR.", "how": "consensus", "rationale": "The notes do not restate why per-item callbacks are better (the slides are not reproduced); delegates simply supported the change. For cleanupSome, optionality gives hosts the only power they need without blocking the rest of WeakRefs, and the web-thread question belongs to the HTML spec.", "dissent": "WebKit kept strategic misgivings about enabling long-running code on the main thread; MM's agreement covered only a presence-or-absence form of optionality.", "confidence": "stated", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "mixed", "shapeWhy": "A yes/no design change coupled with a choice among availability policies for one method.", "secondaryShape": "choice", "fitsFlows": ["ask-teammate", "settle", "revisit-reopen"], "gaps": ["nested follow-ups: Two sub-decisions with different outcomes were settled in one item, one at once and one the next day; a question has one answer, with no settle-part-now or child questions that settle independently.", "external authority: The decisive objection came from someone who could not be persuaded because the decision sits with an absent authority (his organization's leadership); no way to record that persuasion was not the path.", "consensus mechanics: Consent was conditional on exact wording (optional only in the sense of present or absent); stances have a reason but no scoped condition to verify in the final text.", "off-platform decision: Part of the problem was handed to another standards venue (the HTML spec) with named stakeholders to follow up; there is no cross-venue handoff.", "off-platform decision: A negotiated side conversation between days with a promised report back inside the same meeting.", "deadline: Implementation and a Test262 PR were already waiting, so the decision carried shipping urgency that the design cannot express."]}, "evalUse": "Stage 3 proposal with a late implementer objection, a split item and a compromise that delegates part of the question to another venue.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "Both sections read in full. Queue items attributed to a delegate from Stripe and to MM appear as untagged lines at the end of the first section; the participant count includes that delegate. participants.count and process.comments are lower bounds from counting speaker tags in the transcript. Original decider wording (v0): consensus of committee (unanimity; any delegate can block)."} +{"id": "tc39-2020-11-item-method-rename-to-at", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/tc39/notes/blob/main/meetings/2020-11/nov-17.md#item-rename--revisit-inclusion-on-string", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}], "repo": "tc39/notes", "domain": "naming", "title": "Rename the relative-indexing method after web breakage and reaffirm Stage 3", "question": "Since the name item broke real websites, should the relative-indexing method be renamed (starting with at), still be added to String, and stay at Stage 3?", "questionExplicit": true, "context": "Firefox shipped the method unflagged in Nightly and found breakage in a paid 360-degree viewer library, two YUI versions and a photo site, all of which detect DOM collections by the presence of an item property. That removed the original motivation of aligning arrays with DOM collections, leaving only ergonomics for negative indexing.", "options": [{"label": "Rename to at, with itemAt as the next fallback, and keep Stage 3", "proposedBy": "SYG (champion)", "fate": "chosen"}, {"label": "Abandon the feature since the DOM motivation is gone", "proposedBy": "MM", "fate": "rejected"}, {"label": "Demote to Stage 2 and explore new syntax instead", "proposedBy": "JHX", "fate": "rejected"}, {"label": "Prepare a prioritized set of candidate names to try in turn", "proposedBy": "MS, MLS", "fate": "merged"}, {"label": "Leave String out of the proposal", "proposedBy": "KG (mild opposition)", "fate": "rejected"}, {"label": "Throw on out-of-bounds indexes", "proposedBy": "DRR", "fate": "deferred"}], "criteria": ["web compatibility of the name", "ergonomics versus slice and length-minus-one idioms", "code-unit semantics on strings", "consistency across indexable types", "not chasing names indefinitely"], "participants": {"count": 16, "roles": ["champion (V8)", "implementers (Firefox, JavaScriptCore, Chrome)", "chairs", "skeptic of the feature", "proposers of syntax alternatives", "editors"], "decider": "consensus"}, "process": {"research": "survey", "spike": "experiment", "poll": true, "escalation": "none", "subQuestions": ["Which name?", "Include on String?", "Do the feature at all?", "Stay at Stage 3 or go back to Stage 2?", "Syntax instead of a method?", "Throwing out-of-bounds semantics?"], "deferred": false, "reopened": true, "superseded": false, "durationDays": 0, "comments": 67, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["Three non-binding emoji temperature checks steered the order of questions: roughly nine in favour to four unconvinced on doing the feature at all, then a check favouring String inclusion, then one on the whole package", "The champion refused to postpone for a rival syntax idea", "MM said he no longer liked the feature but would not withdraw his consent", "String inclusion had been discussed at the previous meeting and was brought back because delegates wanted more time"]}, "outcome": {"decided": true, "decision": "Consensus to keep Stage 3 for the method on Array, TypedArray and String, pending a rename; try at first and itemAt second.", "how": "consensus", "rationale": "Web compatibility of a short common name can only be tested by shipping, which happens at Stage 3; relative indexing is still worth having and code-unit indexing is already entrenched.", "dissent": "MM still opposed the feature and JHX still wanted a syntax proposal; KG mildly opposed adding it to String; none blocked.", "confidence": "stated", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "mixed", "shapeWhy": "A name choice plus a compatibility claim that only shipping can verify.", "secondaryShape": "check", "fitsFlows": ["research-or-spike", "settle", "revisit-reopen"], "gaps": ["staged acceptance: Staged acceptance: consent for a Stage 3 proposal whose key detail (the name) is verified only by shipping to real users, with an ordered fallback queue and an implicit reopen if breakage is reported.", "evidence beyond check: Evidence came from outside the team (a browser's Nightly bug reports) and arrives over weeks; Research and Spike are short Chopin-run actions, and Reopen is manual.", "formal vote: Informal non-binding temperature checks with counts were used three times to steer the discussion; the design has per-person stances but no lightweight poll that is explicitly not consent.", "blocking dependency: Coupled sub-questions were ordered by dependency (do it at all, then String, then stage, then name); questions are independent and have no ordering or depends-on link.", "consensus mechanics: A participant disliked the outcome but kept his consent in place, which is exactly the stand-aside stance the vocabulary retires as Can live with it.", "other: An earlier agreement on String inclusion was revisited only for lack of discussion time, not because of new information; Reopen does not distinguish the two."]}, "evalUse": "Web-compat rename discovered by shipping, with polls, stage reaffirmation and a collapsed motivation; good for testing provisional settlement.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "Section read in full. Speaker tags include probable transcription duplicates (Jack and JWK, WW and WH), so the count of 16 removes two likely duplicates. The earlier meeting's discussion of String inclusion was not read. participants.count and process.comments are lower bounds from counting speaker tags in the transcript. Original decider wording (v0): consensus of committee (unanimity; any delegate can block)."} +{"id": "tc39-2021-08-realm-constructor-shadowrealm-name", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/tc39/notes/blob/main/meetings/2021-08/aug-31.md#realms-renaming-bikeshedding-thread", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}], "repo": "tc39/notes", "domain": "naming", "title": "Final name for the Realm constructor", "question": "Should the Stage 3 Realm constructor be renamed ShadowRealm so the bikeshedding stops and implementations can proceed?", "questionExplicit": true, "context": "After the callable-boundary redesign the name Realm no longer matched the feature and clashed with terms in web implementations. A public naming thread had run since mid-July 2021, and the unresolved name was holding up implementation.", "options": [{"label": "ShadowRealm", "proposedBy": "LEO (for the champion group)", "fate": "chosen"}, {"label": "Other candidates from the long GitHub bikeshed thread (not listed in the notes)", "proposedBy": "community and champions", "fate": "rejected"}], "criteria": ["meaningful", "unique and searchable", "easy to say and spell", "available without a web clash", "low confusion with Shadow DOM"], "participants": {"count": 5, "roles": ["champion", "implementer (V8, Mozilla)", "delegate raising a naming concern", "chair"], "decider": "consensus"}, "process": {"research": "links", "spike": "none", "poll": false, "escalation": "none", "subQuestions": ["Does the Shadow prefix confuse people who know Shadow DOM?"], "deferred": false, "reopened": false, "superseded": false, "durationDays": 48, "comments": 16, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["The champions picked a single finalist after a seven-week external bikeshed and asked the room not to suggest new names", "An implementer said enough time had been spent and supported closing the bikeshed", "The chair summed up consensus as a compromise nobody loves"]}, "outcome": {"decided": true, "decision": "Consensus on ShadowRealm as the constructor name.", "how": "consensus", "rationale": "The name best met the champions' list of criteria, nothing better had emerged in weeks of discussion, and settling it unblocks implementation.", "dissent": "JHD worried about confusion with Shadow DOM but did not block; no objection was recorded.", "confidence": "implied", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "Naming among candidates, judged on a short criteria list.", "secondaryShape": null, "fitsFlows": ["find", "ask-teammate", "settle"], "gaps": ["off-platform decision: Options were generated off-platform over seven weeks; the meeting saw one finalist, and the choice table assumes options are entered on the card, with no import of an external candidate list or a finalist state.", "deadline: The thread ended because patience ran out and a delegate called time; there is no discussion-has-run-its-course affordance or deadline on a question.", "other: Criteria were a proposer's checklist judged subjectively, so the check/half/cross matrix would imply a rigor the decision did not have.", "consensus mechanics: Everyone treated the outcome as a compromise; Confidence is one value from Chopin and cannot show that no stakeholder is happy.", "blocking dependency: The name was blocking other people's implementation work, a blocked-by relationship to external work that questions cannot show.", "consensus mechanics: One delegate raised a possible collision without proposing an alternative or a block, which fits no stance."]}, "evalUse": "Short naming decision whose real deliberation happened elsewhere; tests whether a tool can represent an external bikeshed and a champions-proposed finalist.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "Section read in full; the bikeshed thread itself was not read. The duration is from the delegate's remark that the thread started on 14 July to the meeting date. participants.count and process.comments are lower bounds from counting speaker tags in the transcript. Original decider wording (v0): consensus of committee (unanimity; any delegate can block)."} +{"id": "tc39-2021-08-pipeline-operator-stage2-hack-style", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/tc39/notes/blob/main/meetings/2021-08/aug-31.md#pipeline-operator-for-stage-2", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}], "repo": "tc39/notes", "domain": "language-design", "title": "Advance the Hack-style pipeline operator to Stage 2 despite a bind-operator condition", "question": "Should the pipeline operator, now with Hack-style topic references, advance to Stage 2 even though an earlier Stage 1 understanding tied its progress to a viable method-extraction (bind-like) proposal?", "questionExplicit": true, "context": "After years of dispute between F#-style and Hack-style pipes, the champions converged on Hack-style with a percent placeholder, with draft spec text and a Babel implementation. In 2017 a delegate had noted that advancing pipeline would kill the bind operator unless a viable successor existed.", "options": [{"label": "Advance to Stage 2 now with Hack-style pipes and percent as the placeholder", "proposedBy": "TAB, JSC (champions)", "fate": "chosen"}, {"label": "Hold pipeline until a viable method-extraction proposal exists", "proposedBy": "JHD (CZW asked the related question about Extensions)", "fate": "rejected"}, {"label": "Advance conditionally, requiring a bind-like proposal to reach Stage 2 later", "proposedBy": "JSC", "fate": "rejected"}, {"label": "Choose another placeholder token (hash clashes with tuples, question mark with partial application)", "proposedBy": "TAB", "fate": "deferred"}], "criteria": ["parseability of the placeholder (edge cases probed live by WH)", "coexistence with the Extensions and bind proposals", "ecosystem demand (State of JS 2020 ranking)", "long-term language impact for modest benefit", "readability gains from linearization"], "participants": {"count": 11, "roles": ["champions", "spec editor reviewing grammar", "delegate holding a prior condition", "Mozilla delegate with reservations", "Extensions champion", "chair"], "decider": "consensus"}, "process": {"research": "survey", "spike": "implementation-pr", "poll": false, "escalation": "none", "subQuestions": ["Which placeholder token?", "May the placeholder be used inside nested functions?", "Does advancing pipeline foreclose method extraction?"], "deferred": false, "reopened": false, "superseded": false, "durationDays": 0, "comments": 62, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["JHD said he would not block if he were the only objector and nobody saw a syntactic obstacle to method extraction", "JHD declined a conditional deal because the expectations attached to an earlier conditional agreement on another proposal had changed afterwards", "YSV voiced strong doubts about long-term value but would not block against broad support", "The chair declared consensus once nobody was willing to block, at the end of the time box"]}, "outcome": {"decided": true, "decision": "Consensus to advance to Stage 2 with reservations on record; the champions are to talk offline with the delegates who still have concerns.", "how": "consensus", "rationale": "Champions had converged on one design, WH found the syntax questions solvable, developer demand for some pipe was strong, and no one was willing to block.", "dissent": "Mozilla (YSV) doubted the benefit against the long-running cost; JHD still wanted a viable syntactic method-extraction path.", "confidence": "implied", "reversibility": "unknown", "implementedIn": []}, "chopinFit": {"shape": "mixed", "shapeWhy": "A go or no-go verdict coupled to a commitment about a different proposal.", "secondaryShape": "check", "fitsFlows": ["ask-teammate", "settle", "revisit-reopen"], "gaps": ["coupled decisions: Coupling across questions: advancing one proposal kills another's path, so a decision's real consequence sits in a different question owned by different people.", "conditional decision: A condition attached at an earlier meeting resurfaced years later; Reopen revisits settled answers but nothing carries forward obligations created by earlier settlements.", "consensus mechanics: Consensus with reservations: two delegates declared they would not block but recorded doubts. Agrees, Objects and Unsure cannot express would block if alone versus will not block, the distinction that decided this outcome.", "other: Trust history shaped the outcome: a participant cited an earlier conditional agreement whose terms later changed, so he refused a conditional deal; informal commitments have no representation.", "evidence beyond check: A reviewer probed grammar edge cases live while the stage decision was open; the check was performed by a human peer, not by Research or Spike.", "consensus mechanics: Consensus was inferred from the time box running out with no blocker, without an explicit call for unwilling-to-block."]}, "evalUse": "Language-design stage advancement that passes with reservations and a cross-proposal obligation; good for testing block versus tolerate semantics.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "Section read in full. The 2017 and 2020 meetings referenced in the debate were not read. Handles in the notes are initials as published. participants.count and process.comments are lower bounds from counting speaker tags in the transcript. Original decider wording (v0): consensus of committee (unanimity; any delegate can block)."} +{"id": "tc39-2022-01-stage3-normative-changes-need-consensus", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/tc39/notes/blob/main/meetings/2022-01/jan-24.md#process-clarification", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}], "repo": "tc39/notes", "domain": "process-governance", "title": "Normative changes to Stage 3 proposals need consensus and agenda notice", "question": "Should the process document state that normative changes to Stage 3 proposals require committee consensus and must be added to the agenda by the usual deadline?", "questionExplicit": true, "context": "PFC presented two PRs, one originally from a Test262 contributor, to formalize what the Temporal champions had been doing as a courtesy to implementers when changing a Stage 3 proposal.", "options": [{"label": "Require consensus and the same agenda deadline as stage advancements, with exceptions expected", "proposedBy": "PFC", "fate": "chosen"}, {"label": "Require consensus but keep the deadline as a courtesy only, since urgent fixes cannot wait two months", "proposedBy": "JHD", "fate": "merged"}, {"label": "Leave urgent cases to ad hoc handling outside plenary", "proposedBy": "SYG", "fate": "merged"}], "criteria": ["reviewer time and burden", "urgency of web-compat fixes", "implementers' need for notice", "ability for the plenary to grant exceptions"], "participants": {"count": 7, "roles": ["Temporal champion", "chair", "implementer (V8)", "object-capability delegate", "other delegates"], "decider": "consensus"}, "process": {"research": "none", "spike": "none", "poll": false, "escalation": "none", "subQuestions": ["Can a hard deadline delay an urgent fix?", "Must the plenary be able to grant an exception, and by consensus?"], "deferred": false, "reopened": false, "superseded": false, "durationDays": 0, "comments": 38, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["MM said he would strongly object unless the rule preserved the plenary's power to grant exceptions; the champion confirmed it did and agreed to say so explicitly", "Consensus was recorded as modulo softer language and encouraging exceptions"]}, "outcome": {"decided": true, "decision": "Consensus to write into the process that normative edits to Stage 3 proposals need committee consensus, and should be on the agenda by the same deadline used for stage advancement, with softened wording so that exceptions are expected and encouraged.", "how": "consensus", "rationale": "It matches current practice, puts the burden on the proposer rather than reviewers, and exceptions cover urgent cases; truly urgent shipping problems are product decisions made outside the plenary.", "dissent": null, "confidence": "stated", "reversibility": "implied", "implementedIn": ["https://github.com/tc39/process-document/pull/32", "https://github.com/tc39/agendas/pull/1073"]}, "chopinFit": {"shape": "choice", "shapeWhy": "Variants of a rule text compared on burden and urgency.", "secondaryShape": null, "fitsFlows": ["settle", "build-pr"], "gaps": ["conditional decision: Approval was conditional on wording changes made afterwards; the design separates settling from landing the PR, but cannot hold settled-subject-to-edits and later verify the PR matches.", "nested follow-ups: The rule contains its own exception clause, which is itself a decision (who may grant exceptions, by what vote).", "consensus mechanics: Consensus came from silence after the chair's call; Agrees, Objects and Unsure expect named stances, so tacit supporters are invisible.", "off-platform decision: One decision touched two repositories (process document and agendas); Built into lists PRs but not their relationship.", "convention scope: The decision governs how other decisions are made; questions cannot be marked as meta-rules that constrain settlement elsewhere."]}, "evalUse": "Small process-governance decision with a conditional objection resolved by drafting an exception; good for testing settle-subject-to-wording.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "Section read in full. The same meeting has a separate item on discouraging polyfills that modify globals at Stage 1 (consensus to revise the process table cell, wording to come back later); it is not part of this record. participants.count and process.comments are lower bounds from counting speaker tags in the transcript. Original decider wording (v0): consensus of committee (unanimity; any delegate can block)."} +{"id": "tc39-2023-01-explicit-support-consensus-procedure", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/tc39/notes/blob/main/meetings/2023-01/jan-30.md#a-procedure-for-multiple-active-supporters-in-committee-to-achieve-consensus", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}], "repo": "tc39/notes", "domain": "process-governance", "title": "Require explicit support from non-champions and invite non-blocking concerns", "question": "Should TC39 change how it gathers consensus so that several delegates beyond the champions state support, and the chairs invite concerns that are explicitly not blocks?", "questionExplicit": true, "context": "DE proposed a How We Work change because consensus currently means no one raised a blocking objection. Silence is ambiguous, especially in remote meetings, and raising any reservation can be read as a threat to block.", "options": [{"label": "At least two delegates outside the champion group state support, and the chairs also solicit non-blocking concerns", "proposedBy": "DE", "fate": "chosen"}, {"label": "A single outside supporter", "proposedBy": "earlier PR discussion (unnamed)", "fate": "rejected"}, {"label": "Count champions toward the minimum", "proposedBy": "KG (raised as a drawback of excluding them)", "fate": "rejected"}, {"label": "Formal nominate-and-second ceremony", "proposedBy": "DE (floated)", "fate": "rejected"}, {"label": "Require supporters to give reasons", "proposedBy": "DE", "fate": "partly"}, {"label": "Limit objections to concerns relevant to the current stage (text in the draft PR)", "proposedBy": "DE (draft)", "fate": "rejected"}, {"label": "Change the underlying unanimity rule", "proposedBy": "MLS (observed it is unanimity in practice)", "fate": "deferred"}], "criteria": ["inclusion and safety to voice doubts", "interpretability of silence", "burden on champions and on large champion groups", "velocity", "preserving the ability to block", "keeping ceremony minimal"], "participants": {"count": 10, "roles": ["proposer", "implementers", "spec editors", "chairs", "object-capability delegate", "other delegates"], "decider": "consensus"}, "process": {"research": "none", "spike": "none", "poll": false, "escalation": "none", "subQuestions": ["How many supporters?", "Do champions count?", "Must supporters give reasons?", "Is any ceremony needed?", "May objections be limited by stage?"], "deferred": false, "reopened": false, "superseded": false, "durationDays": 0, "comments": 67, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["The proposal was self-referential: the room was asked for explicit support and non-blocking concerns on the proposal that introduces them", "The text limiting objections was dropped after WH pointed out it would prevent late-stage rejection of a bad idea", "MLS pointed out that TC39 consensus is in practice unanimity and a veto", "YSV gave a recorded non-blocking concern that the practice could fade back into silence"]}, "outcome": {"decided": true, "decision": "A version of the procedure was adopted: several non-champion delegates must state support during discussion or in a closing solicitation, and the chairs ensure room for non-blocking negative feedback, without nominate-and-second formalities. The objection-limiting paragraph is replaced by a pointer to the process document, and DE lands the updated PR asynchronously.", "how": "consensus", "rationale": "Explicit support reduces the ambiguity of silence, rewards presenters and surfaces tacit doubts at a low cost; it works within the existing unanimity system.", "dissent": "Non-blocking concerns only: YSV worried the practice could decay into silence and that large champion groups may struggle to find two outsiders; DMP found the topic complex.", "confidence": "stated", "reversibility": "implied", "implementedIn": ["https://github.com/tc39/how-we-work/pull/122"]}, "chopinFit": {"shape": "choice", "shapeWhy": "Variants of a procedure compared on inclusion, burden and ceremony.", "secondaryShape": null, "fitsFlows": ["ask-teammate", "settle", "build-pr"], "gaps": ["consensus mechanics: The decision is about how decisions are made. The design assumes one settler pressing Settle; it has no N-of-M explicit-support threshold, champions excluded from the count, or quorum.", "consensus mechanics: The needed vocabulary is blocking objection, non-blocking concern, explicit support and silence; Agrees, Objects and Unsure has no concern-but-will-not-block or support-with-reason, and the record does not list who explicitly supported.", "consensus mechanics: Whether supporters must state reasons was itself debated; Takes require a reason by default.", "staged acceptance: Details were finalized after the meeting by the proposer in a GitHub PR; settled in the room but the text is still open, so the record carries open edits.", "consensus mechanics: The problem described is silence read as consent in remote meetings, and Waiting for Luke could reproduce the same pattern without timeouts or nudges.", "other: The proposal was settled under the rules it proposes, a self-referential case."]}, "evalUse": "Meta-decision about consensus itself, with several coupled design points; good for testing representation of blocks, concerns and explicit support.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "Section read in full. The meeting date (30 January 2023) is taken from the file name and folder. participants.count and process.comments are lower bounds from counting speaker tags in the transcript. Original decider wording (v0): consensus of committee (unanimity; any delegate can block)."} +{"id": "tc39-2023-02-import-assertions-demoted-to-stage2", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/tc39/notes/blob/main/meetings/2023-01/jan-31.md#problems-with-import-assertions-for-module-types-and-a-possible-general-solution--downgrade-to-stage-2", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}, {"url": "https://github.com/tc39/notes/blob/main/meetings/2023-01/feb-02.md#import-assertions", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}], "repo": "tc39/notes", "domain": "compatibility", "title": "Demote import assertions to Stage 2 after web-integration problems", "question": "Should import assertions go back from Stage 3 to Stage 2 so semantics and keyword can change to meet HTML's needs, and what should implementers who have or have not shipped do meanwhile?", "questionExplicit": true, "context": "Integrating with HTML showed that assertions which must not change how a module is interpreted cannot drive request headers or CSP for JSON and CSS modules. Chrome had shipped assert for well over a year, and Node, Deno and much tooling had adopted it.", "options": [{"label": "Demote to Stage 2 with the expectation that the module type will drive interpretation, keeping in-band configuration", "proposedBy": "NRO (champion)", "fate": "chosen"}, {"label": "Stay at Stage 3 and change the proposal by consensus", "proposedBy": "NRO (presented as the alternative); BT (weak preference)", "fate": "rejected"}, {"label": "Ask all implementations to unship the current behaviour", "proposedBy": "JHD", "fate": "rejected"}, {"label": "Ask implementations that have not shipped to hold off", "proposedBy": "NRO, SYG, YSV", "fate": "chosen"}, {"label": "Keep the assert keyword and only relax the interpretation restriction", "proposedBy": "SYG, JRL", "fate": "partly"}, {"label": "Move configuration out of band (CSP, import maps)", "proposedBy": "JHD", "fate": "rejected"}], "criteria": ["integration with HTML fetch and CSP", "developer mental model of what the syntax does", "churn for tooling if the keyword changes", "signal sent to developers and implementers", "scope of what a demotion reopens", "web compatibility of unshipping"], "participants": {"count": 12, "roles": ["proposal champions", "implementers (Chrome, Firefox, Safari, Deno)", "bundler author", "object-capability delegate", "chair", "other delegates"], "decider": "consensus"}, "process": {"research": "links", "spike": "none", "poll": false, "escalation": "none", "subQuestions": ["Does Stage 2 reopen in-band versus out-of-band configuration?", "Should existing implementations unship?", "Change the keyword?", "What forms part of the cache key?"], "deferred": false, "reopened": true, "superseded": false, "durationDays": 2, "comments": 201, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["The first session ended with the demotion requested but no consensus, because delegates disagreed on what Stage 2 would reopen; the item returned two days later", "Chrome said it would not unship and WebKit said it would not ship at Stage 2 but probably would at Stage 3, so the stage was also a message to implementers", "JRL and JHD each said they would not block the demotion despite disagreeing about its scope", "YSV said the Interop 2023 effort was going to drop the feature from its list to avoid sending the wrong signal"]}, "outcome": {"decided": true, "decision": "Consensus to demote to Stage 2 with a shared understanding that semantics must change so the module type drives interpretation. The champions asked others not to ship and did not ask existing implementations to unship, and planned to return in two to four months with a Stage 3 proposal on keyword and cache-key semantics.", "how": "consensus", "rationale": "The web platform needs the type to affect fetching and CSP, and Stage 2 clearly tells developers the feature is unstable and tells non-shipping implementers to wait.", "dissent": "JRL wanted to keep the current syntax in Stage 2; JHD wanted all earlier alternatives, including out of band, on the table; both said they would not block.", "confidence": "stated", "reversibility": "stated-hard", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "Stay at Stage 3 or demote, with the scope of the reopening as a coupled question.", "secondaryShape": "check", "fitsFlows": ["settle", "revisit-reopen"], "gaps": ["reversal after shipping: A demotion is a scoped reopen: what stays settled (the problem to solve, in-band configuration) and what is reopened (keyword, cache key). Reopen returns a question wholesale to open, with no boundaries on the revisit.", "consequences and obligations: The thing being reversed was already in production in several engines and tools; the decision includes messages to implementers and developers (do not ship, do not rely) that the one-way Built into section does not model.", "consequences and obligations: The decision's main effect is outward-facing communication and the notes were asked to carry a particular statement; Settle writes a sentence into the plan but nothing requires an external announcement.", "provisional decision: The first attempt closed in an in-between state (remain at Stage 3 with a shared understanding that changes are coming), neither settled nor open.", "external authority: The real deciders about shipping sit with implementers who said they would not follow a committee recommendation, so the committee cannot settle what it purports to.", "consensus mechanics: Veto semantics are asymmetric: non-champions threatened blocks on the scope of the revisit while the champions held the direction."]}, "evalUse": "Reversal of a Stage 3 feature already shipped, decided over two sessions with a negotiated reopen scope; good for testing reopen and signalling.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "The January session was read in full. The 2 February session was read in the opening, a long middle stretch and the closing resolution, not every turn. Later steps are outside this record: in March 2023 the proposal returned to Stage 3 under the name import attributes with a with keyword and a legacy assert form (read only in part), and in July 2024 assert was dropped after Chrome and Node unshipped it (read in full). participants.count and process.comments are lower bounds from counting speaker tags in the transcript. Original decider wording (v0): consensus of committee (unanimity; any delegate can block)."} +{"id": "tc39-2023-05-array-grouping-static-methods", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/tc39/notes/blob/main/meetings/2023-05/may-16.md#arrayprototypegroup-rename-for-web-compatibility", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}, {"url": "https://github.com/tc39/notes/blob/main/meetings/2023-05/may-18.md#continuation-arrayprototypegroup-rename-for-web-compatibility", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}], "repo": "tc39/notes", "domain": "compatibility", "title": "Array grouping: give up renaming the prototype method, use static methods, demote to Stage 2", "question": "After repeated web-compatibility failures of the Array.prototype grouping names, should the proposal switch to static Object.groupBy and Map.groupBy, and drop to Stage 2 while the changed spec is reviewed?", "questionExplicit": true, "context": "Renaming the instance method earlier in 2022 still hit sites that probe for a group property. The champion JRL no longer had access to do name-availability scans. In November 2022 the committee leaned slightly towards static methods without consensus, and a spec for them was written but never brought back.", "options": [{"label": "Rename the instance methods again to very obscure names (groupToObject and groupToMap)", "proposedBy": "DE, KG", "fate": "rejected"}, {"label": "Static methods Object.groupBy and Map.groupBy over any iterable", "proposedBy": "JRL, JHD", "fate": "chosen"}, {"label": "Never add new Array.prototype methods", "proposedBy": "DLM (Mozilla preference), DE (acceptable)", "fate": "partly"}, {"label": "Stay at Stage 3 with a co-champion and review the new spec in an overflow slot", "proposedBy": "JHD", "fate": "rejected"}, {"label": "Demote to Stage 2 now and return for Stage 3 at the next meeting", "proposedBy": "EAO, MLS", "fate": "chosen"}, {"label": "Make grouping an iterator helper", "proposedBy": "PKA", "fate": "rejected"}], "criteria": ["web compatibility of names", "Mozilla's appetite for new Array.prototype methods", "strength of the Stage 3 signal to implementers", "champion availability", "whether the changed spec had been reviewed", "consistency with other iterable-consuming APIs"], "participants": {"count": 11, "roles": ["champion and incoming co-champion", "implementers (Mozilla, V8, JavaScriptCore)", "spec editor", "outside delegates", "chair"], "decider": "consensus"}, "process": {"research": "data-analysis", "spike": "implementation-pr", "poll": false, "escalation": "none", "subQuestions": ["Is adding Array.prototype methods still possible at all?", "What happens to Stage 3 proposals whose champion is unavailable?", "Should a proposal be demoted and re-promoted at the next meeting?", "Should Map.groupBy construct a Map directly?"], "deferred": false, "reopened": true, "superseded": false, "durationDays": 2, "comments": 74, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["The first session produced a co-champion and an overflow slot but no stage change", "On the second day the new spec turned out to be materially different (static, iterable-taking), which triggered demotion", "Mozilla said it would resist further new Array.prototype methods, with an exception for naturally obscure names", "Delegates endorsed demoting and re-promoting at the very next meeting as a normal pattern, citing import attributes"]}, "outcome": {"decided": true, "decision": "The proposal drops to Stage 2 with static methods on Object and Map (Map built directly, iterable input); JHD co-champions, reviewers are KG, SYG and a Safari delegate to be recruited, and Stage 3 is to be re-requested at the following meeting.", "how": "consensus", "rationale": "Names kept colliding with real sites, and the new spec differed enough from the Stage 3 text that the Stage 3 criteria were no longer met.", "dissent": "No objection to the demotion was recorded; Mozilla's broader reluctance about new Array.prototype methods remained.", "confidence": "stated", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "Obscure instance names versus static methods, with a stage consequence.", "secondaryShape": "check", "fitsFlows": ["ask-teammate", "settle", "revisit-reopen"], "gaps": ["provisional decision: A chain of provisional answers across at least four meetings (group to groupBy rename, compat bug, static-methods lean, this demotion), each invalidated by field evidence; F6 covers one reopen, not a series where each settlement was explicitly provisional.", "delegated decider: Ownership changed in the room: a co-champion was added because the champion lacked time; Settled by is one person and the design has no champion or owner role or handoff.", "evidence beyond check: The evidence was obscure breakage reports that were hard to trace to the cause; nothing lets evidence be attached with its quality and expiry.", "convention scope: One decision generated a general policy stance (resist new Array.prototype methods) that constrains future questions; there is no place for precedent that applies elsewhere.", "conditional decision: Acceptance criteria for the next step were agreed up front (reviews of the new spec, return next meeting), and delegates asked for them to be written down.", "consequences and obligations: The Stage 3 signal is one bit for implementers, so demotion is partly a communication act, not a design change."]}, "evalUse": "Web-compat rename that fails twice and ends in a different design plus demotion; shows a multi-meeting chain and champion handoff.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "Both sessions read in full. Earlier steps (the 2022-06 rename of groupBy to group, the 2022-11 web-compat discussion) and the later 2023-09 update were not read. The proposal repository is linked in the notes but its issues were not read. participants.count and process.comments are lower bounds from counting speaker tags in the transcript. Original decider wording (v0): consensus of committee (unanimity; any delegate can block)."} +{"id": "tc39-2023-07-design-freeze-stage-before-stage3", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/tc39/notes/blob/main/meetings/2023-07/july-13.md#reducing-wasted-effort-due-to-proposal-churn", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}, {"url": "https://github.com/tc39/notes/blob/main/meetings/2023-07/july-13.md#reducing-wasted-effort-due-to-proposal-churn-continuation", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}], "repo": "tc39/notes", "domain": "process-governance", "title": "Add a design-freeze stage between Stage 2 and Stage 3 (no change made)", "question": "Should TC39 add a stage after Stage 2 where the design is frozen and tests are written before the proposal is recommended for implementation, and what should the stages be called?", "questionExplicit": true, "context": "MF argued that writing Test262 tests and implementing both uncover design problems that force rework after Stage 3, while demanding tests before Stage 3 means redoing them if the design still moves. Outsiders also treat Stage 3 as ready to ship, which makes renumbering costly.", "options": [{"label": "New stage between 2 and 3 where design is signed off and tests get written (working name two and three-quarters)", "proposedBy": "MF", "fate": "deferred"}, {"label": "Only require Test262 tests before Stage 3, with no new stage", "proposedBy": "EAO", "fate": "deferred"}, {"label": "Keep the current stages and be more diligent about tests during Stage 3", "proposedBy": "MLS", "fate": "deferred"}, {"label": "Keep Stage 3 as the sign-off and add a later three-and-a-quarter stage once tests are done", "proposedBy": "WH", "fate": "deferred"}, {"label": "Formalize criteria for demoting Stage 3 proposals after substantial implementer feedback", "proposedBy": "SFC", "fate": "deferred"}, {"label": "Develop very large proposals like Temporal through an incubation or spiral model instead of design-up-front", "proposedBy": "SYG (as a follow-up conversation)", "fate": "deferred"}], "criteria": ["aggregate effort across champions, test writers and implementers", "clarity of the Stage 3 signal to the wider community", "where design sign-off and reviews happen", "delay for small proposals", "barrier to writing Test262 tests", "fit for very large proposals"], "participants": {"count": 14, "roles": ["champion of the process change", "spec editors", "implementers (V8, SpiderMonkey, JavaScriptCore)", "Test262 maintainer", "Intl delegates", "chairs"], "decider": "consensus"}, "process": {"research": "none", "spike": "none", "poll": false, "escalation": "none", "subQuestions": ["Naming and numbering of the new stage", "Who writes the tests", "Should it apply to proposals already at Stage 3", "Which stage carries reviews and editor sign-off", "Should polyfill or engine262 implementations come earlier", "Criteria for demoting from Stage 3"], "deferred": true, "reopened": false, "superseded": false, "durationDays": 0, "comments": 110, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["Two timeboxed sessions on the same day; the first summary recorded generally positive feedback and no progress on naming, the continuation only repeated it", "An implementer corrected the claim that engines wait for Test262 tests before implementing", "A long exchange between WH and others on whether the committee's design sign-off should stay at Stage 3", "A side thread on formal criteria for demoting Stage 3 proposals"]}, "outcome": {"decided": false, "decision": "The committee left the stage process unchanged; reaction to the idea was generally positive, and further feedback was requested outside the meeting.", "how": "default/no-action", "rationale": "Time ran out with open concerns about added delay, which stage carries design sign-off, and the numbering.", "dissent": "MLS had grave concerns the new stage would add delay without helping; EAO doubted a new stage was needed; WH wanted design sign-off to remain at Stage 3.", "confidence": "stated", "reversibility": "unknown", "implementedIn": []}, "chopinFit": {"shape": "explore", "shapeWhy": "Open-ended process design with many coupled facets, not a bounded set of options.", "secondaryShape": "choice", "fitsFlows": ["ask-teammate", "revisit-reopen"], "gaps": ["coupled decisions: The proposal carried a working name while naming and numbering were unresolved, so a design and its label were bundled and the label could block the design.", "off-platform decision: The discussion needed two sessions on one day with a carried-over queue; a question has no continuation across sessions.", "deferred: The outcome was a sense of the room (generally positive), which is neither Settle nor Open; there is no artefact for direction of travel without a decision.", "evidence beyond check: Factual claims were disputed live (empirical stability after tests, whether implementers wait for tests); a peer corrected a peer, and Check has no way to attach a correction to a claim in a thread.", "consequences and obligations: The costs fall on different roles (champions, test writers, implementers, editors) and the asymmetry is the crux, but Takes have no role or cost dimension.", "nested follow-ups: Transition rules (apply retroactively to current Stage 3 proposals?) are a sub-question with its own audience."]}, "evalUse": "Process proposal that ends with no change and an explicit record of unresolved naming; good for testing deferred or no-decision outcomes.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "Both sessions read in full. Later agenda items in this repository use a Stage 2.7 label (for example 2024-04 and 2025), which suggests a version of this idea was adopted later, but that was not verified in this record. participants.count and process.comments are lower bounds from counting speaker tags in the transcript. Original decider wording (v0): consensus of committee (unanimity; any delegate can block)."} +{"id": "tc39-2023-11-operator-overloading-withdrawn", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/tc39/notes/blob/main/meetings/2023-11/november-28.md#withdrawing-operator-overloading", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}], "repo": "tc39/notes", "domain": "language-design", "title": "Withdraw the operator overloading proposal", "question": "Should the committee explicitly withdraw operator overloading so that Decimal, Records and Tuples and others stop assuming it might exist?", "questionExplicit": true, "context": "DE, the champion, argued engine implementers, loudest among them V8, had repeatedly said they would block operator overloading, and the lexically scoped opt-in design added further cost. He wanted an explicit committee conclusion instead of an ambiguous dormant proposal.", "options": [{"label": "Withdraw by explicit consensus", "proposedBy": "DE", "fate": "chosen"}, {"label": "Leave the proposal dormant without a committee decision", "proposedBy": "status quo DE argued against", "fate": "rejected"}, {"label": "Withdraw, but record that it may be revived if implementer views change", "proposedBy": "MM (condition for support)", "fate": "merged"}, {"label": "Delay the withdrawal until Google's position was confirmed", "proposedBy": "CDA, SFC", "fate": "partly"}, {"label": "Overload Math functions for BigInt instead", "proposedBy": "JSC", "fate": "rejected"}], "criteria": ["implementation cost and complexity", "benefit for numeric types, units and vectors", "value of an explicit committee statement over ambiguity", "reversibility", "effect on dependent proposals"], "participants": {"count": 9, "roles": ["champion", "implementers (SpiderMonkey, Google)", "object-capability delegate", "affected proposal champion", "chair", "other delegates"], "decider": "consensus"}, "process": {"research": "none", "spike": "none", "poll": false, "escalation": "none", "subQuestions": ["Effect on the BigInt Math functions proposal (put them on BigInt instead of overloading Math)", "Was it fair to add the item late without Google having discussed it?"], "deferred": false, "reopened": false, "superseded": false, "durationDays": 0, "comments": 52, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["MM supported only reluctantly, on the condition that the wording keeps the decision revisitable", "A Google delegate said the company had no formal position because the item was added late, even though the main V8 implementer had spoken the day before", "The chair made consensus conditional on later confirmation from the V8 delegate, which arrived after the meeting", "A knock-on decision about another proposal was made within the same item"]}, "outcome": {"decided": true, "decision": "Consensus to withdraw operator overloading, pending Google's confirmation (given after the meeting). It could return later if implementers re-assess its cost and benefit, but the committee does not plan to spend time on it now.", "how": "consensus", "rationale": "Engine teams judged the cost and complexity too high for the benefit, especially with scoping safeguards.", "dissent": "MM reluctantly agreed and regretted losing a principled design; JHD and JSC accepted the loss of ergonomics; no blocking objection.", "confidence": "stated", "reversibility": "stated-easy", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "Withdraw, keep dormant or delay, judged on cost versus benefit.", "secondaryShape": null, "fitsFlows": ["ask-teammate", "settle", "revisit-reopen"], "gaps": ["moot question: A negative decision: the outcome removes a feature and closes a line of work. Settle writes an answer sentence into the plan, and nothing records withdrawn, dropped or not doing it.", "provisional decision: Reversibility was negotiated into the wording by one stakeholder (explicitly revisitable, if implementer positions change); the design has a single Easy to change fact but no named reopen condition.", "external authority: The decisive voice was an absent party (V8's position stated the day before), and settlement was conditional on its after-the-fact confirmation.", "consensus mechanics: A notice-period fairness issue (item added after the agenda deadline) affected whether a stakeholder could take a position.", "coupled decisions: Knock-on effects on other questions (Decimal, BigInt Math) were discussed inside this thread and decided there; there is no dependency link to the affected questions.", "consensus mechanics: The proposer and the party stepping down are the same person, so Settled by does not capture sponsor withdrawal or the weight of implementer vetoes."]}, "evalUse": "Proposal withdrawal driven by implementer positions, with reluctant consent and a conditional confirmation; good for testing negative outcomes and reversibility.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "Section read in full. The previous day's implementer comments were not read. Speaker tags show 9 participants; more people supported on the queue. participants.count and process.comments are lower bounds from counting speaker tags in the transcript. Original decider wording (v0): consensus of committee (unanimity; any delegate can block)."} +{"id": "tc39-2024-04-math-sumprecise-design-points", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/tc39/notes/blob/main/meetings/2024-04/april-09.md#mathsumexact-for-stage-27", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}], "repo": "tc39/notes", "domain": "api-design", "title": "Design points and final name of Math.sumPrecise before Stage 2.7", "question": "What should the precise-summation API accept, be called, return for an empty list and do when it meets NaN, so that it can advance to Stage 2.7?", "questionExplicit": true, "context": "KG proposed a built-in sum of an iterable of numbers that is as precise as possible, argued to be practical because WH had shown a full-precision algorithm and KG had prototyped it in JavaScript. KG brought five open design questions to the stage request.", "options": [{"label": "Name it sumExact", "proposedBy": "KG", "fate": "rejected"}, {"label": "Name it sumPrecise", "proposedBy": "MM", "fate": "chosen"}, {"label": "Name it sumExactFrom to signal iterable input", "proposedBy": "MF", "fate": "rejected"}, {"label": "Return negative zero for an empty list", "proposedBy": "KG", "fate": "chosen"}, {"label": "Return positive zero for an empty list", "proposedBy": "unnamed delegates at the previous meeting", "fate": "rejected"}, {"label": "Drain the whole iterator even after NaN, to keep results independent of order", "proposedBy": "KG", "fate": "chosen"}, {"label": "Exit early on NaN", "proposedBy": "MF", "fate": "rejected"}, {"label": "Variadic arguments like Math.max", "proposedBy": "implicit alternative (Math.max precedent)", "fate": "rejected"}], "criteria": ["developer expectations created by the word exact", "commutativity of results regardless of input order", "performance (champion cited two to four times slower than naive)", "feasibility of the exact algorithm", "stack limits on variadic calls", "precedent in Math.hypot and Math.max"], "participants": {"count": 14, "roles": ["champion", "spec editors", "implementers (JavaScriptCore, SpiderMonkey, V8)", "Intl delegate", "object-capability delegate", "chair"], "decider": "consensus"}, "process": {"research": "links", "spike": "prototype", "poll": true, "escalation": "none", "subQuestions": ["Name", "From suffix", "Zero for an empty list", "NaN handling", "Iterable versus variadic", "Coercion of inputs"], "deferred": false, "reopened": false, "superseded": false, "durationDays": 0, "comments": 77, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["A temperature-check poll on early exit for NaN came out eight unconvinced to five in favour; the champion then decided himself rather than seeking further votes", "The new name was floated by a delegate in the room and adopted immediately", "At a delegate's request the champion restated every decision point at the end for the record", "Many delegates gave non-speaking plus-ones as explicit support"]}, "outcome": {"decided": true, "decision": "Stage 2.7 granted under the name sumPrecise; it takes an iterable, does no coercion, returns negative zero for an empty list, drains the whole iterator after NaN, and the spec is written in real-number arithmetic.", "how": "consensus", "rationale": "The name exact promised more than it delivers and would confuse developers who expect decimal behaviour; draining keeps the result independent of input order; negative zero is the floating-point additive identity.", "dissent": "MF still preferred early exit on NaN and including From, neither blocking.", "confidence": "stated", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "mixed", "shapeWhy": "A bundle of small choices (name, edge values, signature) gated by one stage request.", "secondaryShape": "choice", "fitsFlows": ["research-or-spike", "ask-teammate", "settle"], "gaps": ["nested follow-ups: One proposal bundles about six small coupled decisions that a stage request settles together; the design has one question and one answer, with no checklist question that settles N sub-decisions and ends with a recap.", "formal vote: After a split informal poll (eight against five) the owner made the call himself; the poll counts and champion discretion should be recorded as the method, and Settled by alone does not show it.", "reframing: The option set changed in the room: a non-champion's new name won, and arguments were about what users will expect, not cells on a grid.", "evidence beyond check: The feasibility and performance claims were accepted on the authority of reviewers (an algorithm validity claim from WH, unverified speed figures); Check has no field for who vouched.", "consensus mechanics: Support was expressed through non-speaking plus-ones on a queue, the explicit-support convention the committee had just adopted, with no record of who they were.", "other: Naming reasoning compared a champion's expert view with a naive developer's, which a stance with a reason does not distinguish."]}, "evalUse": "Compact example of many small coupled design decisions resolved with a poll and champion tie-break; good for testing multi-part settlement.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "Section read in full. The earlier meeting at which empty-list and NaN handling were first discussed was not read. Performance figures are the champion's recollection of third-party claims. participants.count and process.comments are lower bounds from counting speaker tags in the transcript. Original decider wording (v0): consensus of committee (unanimity; any delegate can block)."} +{"id": "tc39-2025-09-iterator-windows-undersized-parameter", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/tc39/notes/blob/main/meetings/2025-05/may-29.md#iterator-chunking-for-stage-27", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}, {"url": "https://github.com/tc39/notes/blob/main/meetings/2025-07/july-29.md#iterator-chunking-for-stage-27", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}, {"url": "https://github.com/tc39/notes/blob/main/meetings/2025-09/september-22.md#iterator-chunking-for-stage-27", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:42:41Z"}], "repo": "tc39/notes", "domain": "api-design", "title": "Windowing helper for short iterators: behaviour, one method or two, and parameter shape", "question": "For the iterator windowing helper, what should happen when the source yields fewer items than the window size, should the behaviours be separate methods or one method with a parameter, and what should the parameter look like before Stage 2.7?", "questionExplicit": true, "context": "MF's iterator chunking proposal adds chunks, which yields an undersized last chunk, and windows, which as specified yields nothing for a too-short source. KG found that surprising, and MF compared four behaviours against practice in other languages and libraries over three meetings.", "options": [{"label": "Yield no windows when the source is too short", "proposedBy": "MF (as specified)", "fate": "chosen"}, {"label": "Throw on the first next call", "proposedBy": "MF (alternative PR)", "fate": "rejected"}, {"label": "Yield one undersized window", "proposedBy": "MF (alternative PR)", "fate": "chosen"}, {"label": "Pad the window to full size", "proposedBy": "MF (alternative PR)", "fate": "rejected"}, {"label": "Two methods, windows and sliding", "proposedBy": "MF (May and July)", "fate": "rejected"}, {"label": "One windows method with an optional string parameter", "proposedBy": "KG", "fate": "chosen"}, {"label": "Options bag instead of a string", "proposedBy": "JHD", "fate": "rejected"}, {"label": "Return leftover items in the done result", "proposedBy": "JHD (from a TG3 idea)", "fate": "rejected"}], "criteria": ["precedent in other languages and libraries", "TypeScript typing of fixed-size windows", "silent data loss", "discoverability and autocomplete", "avoiding two near-identical methods", "throwing as an anti-pattern", "room for future options", "actual use cases"], "participants": {"count": 19, "roles": ["champion", "spec editors", "implementers", "TypeScript delegates", "skeptical reviewers", "Intl delegates", "chairs"], "decider": "consensus"}, "process": {"research": "survey", "spike": "implementation-pr", "poll": false, "escalation": "none", "subQuestions": ["Is windows worth adding at all?", "Should chunks change to match?", "String versus options bag", "Default value", "Spelling of the string values (spaces versus kebab-case)", "Should a throwing mode exist later?"], "deferred": true, "reopened": true, "superseded": true, "durationDays": 116, "comments": 182, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["May: a comparison of four behaviours with precedent from other languages; delegates asked who actually uses windows, and peers produced call sites and Rust standard-library usage counts live", "July: the champion presented a two-method design, but reviewer feedback received days earlier from KG and NRO moved the room to a single method, and the parameter shape was left to the issue tracker", "September: the string values were changed at the last minute to kebab-case after learning of a de facto convention, which also prompted a separate same-day discussion about normative conventions", "Stage 2.7 was granted in September with no objections"]}, "outcome": {"decided": true, "decision": "Stage 2.7 for iterator chunking: chunks plus a single windows method with an optional undersized parameter taking only-full (the default) or allow-partial, no throwing mode, kebab-case string values.", "how": "consensus", "rationale": "Both behaviours serve real use cases, two near-identical methods would be confusing, most uses want no output for an incomplete window, throwing is an anti-pattern, and kebab-case follows de facto precedent.", "dissent": "KG leaned slightly towards throwing as the default and JRL preferred allow-partial as default to match chunks; neither blocked. MM and RGN doubted windows is needed.", "confidence": "stated", "reversibility": "implied", "implementedIn": ["https://github.com/tc39/proposal-iterator-chunking/pull/24"]}, "chopinFit": {"shape": "choice", "shapeWhy": "Competing API behaviours compared on precedent, typing and use cases.", "secondaryShape": "check", "fitsFlows": ["research-or-spike", "settle", "revisit-reopen", "build-pr"], "gaps": ["reframing: The question evolved across three plenaries (116 days) with different people each time; options were replaced, not added (two methods to one, spaces to kebab-case), so How it moved needs a snapshot of the proposal at each presentation.", "evidence beyond check: Alternatives were embodied as parallel PRs before the decision, one per option; Spike is a single throwaway built after a hunch, not a PR per option.", "deadline: Feedback that arrived days before a meeting changed the design and the champion said it should have come earlier; the design has no early-input nudge or feedback deadline.", "evidence beyond check: Evidence was demanded in the thread (does anyone use windows?) and produced live by other delegates; Research is Chopin's action and cannot take contributions from peers.", "convention scope: A local naming choice (kebab-case for string values) set a convention that was being formalized the same day; there is no link from a decision to the general rule it instantiates.", "consensus mechanics: Design details were settled in the same agenda item as the stage gate, and a late post-deadline change was accepted in the room, so notice rules affect what can be settled."]}, "evalUse": "API design that changes shape across three meetings with competing implementations; good for testing long-running questions with replaced options.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "May and July items read in full; the September item read except for a few middle turns. The implementedIn URL is built from pull request number 24 mentioned in the July notes and the proposal repository named there; it was not opened. Alternative PR numbers 14 to 16 are mentioned in the May notes. participants.count and process.comments are lower bounds from counting speaker tags in the transcript. Original decider wording (v0): consensus of committee (unanimity; any delegate can block)."} +{"id": "rust-lang-team-2021-07-decision-process-tenets-and-dissent", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/rust-lang/lang-team/blob/c45345e32e81e273aab6c805b0d3cd7b1a666045/design-meeting-minutes/2021-07-21-decision-making-tenets-and-process.md", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:50:00Z"}, {"url": "https://github.com/rust-lang/lang-team/blob/c45345e32e81e273aab6c805b0d3cd7b1a666045/src/fcp-decisions.md", "kind": "other", "retrievedAt": "2026-10-18T05:50:00Z"}, {"url": "https://github.com/rust-lang/lang-team/blob/c45345e32e81e273aab6c805b0d3cd7b1a666045/src/rationale/pr-360.md", "kind": "other", "retrievedAt": "2026-10-18T05:50:00Z"}], "repo": "rust-lang/lang-team", "domain": "process-governance", "title": "How the language team makes decisions, records dissent, and who may block", "question": "Which tenets and bot-supported procedure should the language team use to propose, hold, approve and dissent on decisions, and should it add an advisors subteam?", "questionExplicit": false, "context": "A design-meeting document proposed decision-making tenets, a rustbot command set with a ten-day clock, a split between reversible and irreversible decisions, recorded dissent, and an advisors subteam. The same repository later shows the process revised in April 2026.", "options": [{"label": "Bot-tracked decision periods with per-member statuses (merge, hold, abstain, dissent, close), a ten-day clock from the start, and reversible vs irreversible classes", "proposedBy": "meeting document authors", "fate": "partly"}, {"label": "Keep the existing rfcbot behaviour where the clock starts only after consensus", "proposedBy": "status quo", "fate": "rejected"}, {"label": "Checkbox sign-off per member", "proposedBy": "status quo", "fate": "rejected"}, {"label": "A separate postpone status", "proposedBy": "not proposed; FAQ rejects", "fate": "rejected"}, {"label": "Rename restart to recheck", "proposedBy": "joshtriplett", "fate": "partly"}, {"label": "Advisors subteam with no formal decision authority", "proposedBy": "meeting document authors", "fate": "chosen"}, {"label": "Let advisors and other team members start decisions, and perhaps block", "proposedBy": "Mark-Simulacrum", "fate": "partly"}], "criteria": ["treasure dissent", "do not force irreversible decisions", "value expertise", "record reasoning", "consensus without unanimity", "speed of resolving holds", "autonomy of champions"], "participants": {"count": 6, "roles": ["lang team members"], "decider": "team"}, "process": {"research": "none", "spike": "none", "poll": false, "escalation": "none", "subQuestions": ["what switches a decision between mutable and immutable", "whether members start abstaining or blank", "should non-members be able to invite concerns or start a decision", "when is a dissent written and what follows from it", "should restart exist and what should it be called"], "deferred": true, "reopened": true, "superseded": true, "durationDays": null, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["the tenet about not forcing irreversible decisions was challenged for ignoring the cost of never shipping and was discussed at length", "the restart command was questioned as a mode of operation and a rename suggested", "a later process revision (April 2026) replaced the scheme; a middle revision based on a different bot procedure was never put into practice", "the 2026 revision's decision on whether a concern raiser may also block its resolution was not unanimous and one member's dissent was left to be written later"]}, "outcome": {"decided": false, "decision": "The meeting notes record questions and partial resolutions but no approval. The repository's current docs keep the dissent axiom, split decisions into champion (single person, reviewed afterwards) and team FCP kinds, keep advisors who may propose and raise concerns, and describe sign-off through rfcbot checkboxes.", "how": "unclear", "rationale": "Proponents wanted to make progress possible when a member objects, by recording dissent instead of blocking indefinitely; critics pressed on irreversibility, restart semantics and who may block.", "dissent": "In the 2026 revision, Josh Triplett disagreed with removing the ability of a concern raiser to also block the concern's resolution; the rationale page leaves space for his dissent text to be added in a future PR.", "confidence": "none", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "mixed", "shapeWhy": "A set of related policy choices (tenets, statuses, roles) debated as one document.", "secondaryShape": "explore", "fitsFlows": ["ask-teammate", "settle", "revisit-reopen"], "gaps": ["consensus mechanics: The decision defines the stances themselves: merge, hold, abstain, dissent, close. Chopin's Agrees / Objects / Unsure lacks a hold (paused while someone formulates a concern), a non-blocking recorded dissent, and abstain.", "consensus mechanics: Decisions come in classes (reversible vs irreversible) that change who must sign: all members for stabilisation, most for merge. The design has one Settle button for one settler and no quorum or class.", "consensus mechanics: A running clock (ten days, restart on material change) governs when a decision is final. There is no timer, no pings on stale holds, no restart that clears stances while preserving history.", "consensus mechanics: Recorded dissent is a durable artefact meant to be re-checked at later stages (proposal, RFC, feature complete). The record has no carried-forward dissent or unresolved question attached to a settled answer.", "delegated decider: Decision mode varies by champion (one person, unblockable, reviewed after the fact) vs team. Chopin assumes Ada settles alone.", "superseded or moved to another thread: Revisited five years later across at least two revision PRs, one never adopted, plus a non-unanimous sub-decision with a dissent whose text was promised later. Reopen models one flip, not a lineage of superseded versions, nor 'promised content pending'.", "off-platform decision: The meeting document is the decision text and the discussion is inline questions: no separate question, answer and take objects."]}, "evalUse": "A decision-about-deciding with explicit tenets, many small resolved sub-questions, and a documented revision years later with a recorded dissent.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "All three files were read from the clone. The 2021 meeting text says it is a proposal pending approval; no approval is recorded there, so decided is false. The later documents (champion and FCP decision pages and the PR 360 rationale) show what survived; the fate labels for adopted pieces are inferred from those current docs, and the PRs themselves (lang-team PR 360 and issue 365, both cited in the rationale) were not fetched. participants.count is the number of distinct handles in the notes (Taylor and Mark are first-name forms). The question is implicit in the document framing; sub-questions are explicit headings."} +{"id": "rust-lang-team-2022-03-lint-policy-ownership", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/rust-lang/lang-team/blob/c45345e32e81e273aab6c805b0d3cd7b1a666045/design-meeting-minutes/2022-03-30-lint-policy.md", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:50:00Z"}], "repo": "rust-lang/lang-team", "domain": "tooling", "title": "Which lints the language team should own in the compiler versus delegate to Clippy", "question": "Should the language team stop adding, or even move out, lints that only teach idioms or enforce style, and keep only lints tied to correctness or future compatibility?", "questionExplicit": true, "context": "A team member classified existing lints into four purposes (future-proofing, error heuristics, uniform style, education) and argued the lang team is overextended and should not decide the subjective categories.", "options": [{"label": "Mild: stop adding new education and style lints to the compiler", "proposedBy": "pnkfelix", "fate": "rejected"}, {"label": "Wild: move all existing education and style lints to Clippy", "proposedBy": "pnkfelix", "fate": "rejected"}, {"label": "Keep subjective lints in the compiler for their norming effect, but watch signal-to-noise closely", "proposedBy": "joshtriplett", "fate": "chosen"}, {"label": "Require no lang-team FCP for lints at all, with a second from the owning team when a lint de facto deprecates something", "proposedBy": "scottmcm", "fate": "deferred"}, {"label": "Start from the goals lints serve and work backwards", "proposedBy": "nikomatsakis", "fate": "deferred"}, {"label": "A test that fails on warnings instead of deny-warnings", "proposedBy": "joshtriplett", "fate": "deferred"}], "criteria": ["signal-to-noise ratio", "norming effect across the ecosystem", "lang-team bandwidth", "maintenance burden", "subjectivity", "reversibility of a lint", "default reachability (Clippy is opt-in)"], "participants": {"count": 8, "roles": ["lang team members", "compiler contributors", "observer in call chat"], "decider": "unclear"}, "process": {"research": "none", "spike": "none", "poll": false, "escalation": "none", "subQuestions": ["are the four categories mutually exclusive", "should future-compat and edition lints be grouped", "does the policy apply to allow-by-default lints", "do deny-by-default lints belong in rustc"], "deferred": true, "reopened": false, "superseded": false, "durationDays": 0, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["the author said his own view could readily flip to the opposite side", "a table borrowed from another draft was withdrawn as misleading", "participants proposed multi-tag categories instead of exclusive ones", "the author concluded his specific proposals would not happen and suggested taking the rest to chat"]}, "outcome": {"decided": false, "decision": "The author's two proposals were not taken forward; the team agreed only that categories overlap and that goals should be clarified first, with follow-up moved to chat.", "how": "default/no-action", "rationale": "Several members argued compiler lints have a norming effect that Clippy lacks and that migrating lints out would hurt discoverability and CI stability.", "dissent": null, "confidence": "stated", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "explore", "shapeWhy": "Open framing and categorisation, with a mild and a wild proposal as probes rather than a closed option set.", "secondaryShape": "choice", "fitsFlows": ["ask-teammate", "settle", "reshape"], "gaps": ["deferred: Ended by sense of the room: the proposals will not happen, no poll or record of who said what. The closest state is Discard question, which loses the reasons and the follow-up (continue elsewhere).", "rejected options content: One option was deliberately wild (a provocation). There is no marker for options offered to test the space rather than to be chosen.", "other: The classification itself is contested (labels overlap). Chopin shapes are one per question; this needed multi-label tagging of items, an evaluative table rather than a card.", "delegated decider: Decider unclear: lang, compiler and Clippy ownership overlap, so the question of who may decide is part of the question.", "evidence beyond check: Missing evidence is part of the argument (no telemetry to know whether noise is acceptable). What we knew cannot mark 'we have no data and cannot get it'.", "off-platform decision: Inline comments in the doc, call chat, and a queue of big questions ran in parallel; stances lived in prose and in the chat. Takes are tied to a single Ask flow.", "nested follow-ups: A cross-cutting policy question; many sub-opinions on scoping (allow-by-default, deny-by-default) each with their own mini-conclusions that never get settled."]}, "evalUse": "Explicit non-adoption with overlapping categories, strong stances, and no data; good for testing summarisation of a meeting that ends without a vote.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "Full file read from the clone. A second file (2022-03-30-lints.md) in the same folder has the same length and appears identical in its first lines; a diff printed nothing, so it is a duplicate. Participant count is distinct handles in the notes (including two chat-only comments by the same person). The outcome is the author's own end-of-meeting statement, not a recorded team decision."} +{"id": "rust-lang-team-2022-03-lang-roadmap-themes", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/rust-lang/lang-team/blob/c45345e32e81e273aab6c805b0d3cd7b1a666045/design-meeting-minutes/2022-03-09-lang-roadmap.md", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:50:00Z"}, {"url": "https://github.com/rust-lang/lang-team/blob/c45345e32e81e273aab6c805b0d3cd7b1a666045/src/roadmaps/roadmap-2024.md", "kind": "other", "retrievedAt": "2026-10-18T05:50:00Z"}], "repo": "rust-lang/lang-team", "domain": "process-governance", "title": "Themes, order and audience for the language team's Rust 2024 roadmap", "question": "Which themes should the roadmap lead with, in what order, who is it written for, and what should it ask of readers?", "questionExplicit": true, "context": "A draft roadmap named three themes and gave examples under each. The meeting discussed purpose, scope, ordering, missing items and whether readers know what to do next.", "options": [{"label": "Draft order: help users help each other, flatten the learning curve, help the project scale", "proposedBy": "roadmap authors", "fate": "rejected"}, {"label": "Put project scaling first", "proposedBy": "Mark and lcnr", "fate": "rejected"}, {"label": "Lead with the learning-curve theme, user-ecosystem second, project scaling last (published order)", "proposedBy": "not attributed in notes", "fate": "chosen"}, {"label": "Add a help-wanted section and a concrete next step for each item", "proposedBy": "nikomatsakis, pnkfelix, simulacrum", "fate": "chosen"}, {"label": "Add C++ interop as an item or working group", "proposedBy": "pnkfelix", "fate": "deferred"}, {"label": "Include anti-examples to steer people away from rabbit holes", "proposedBy": "pnkfelix", "fate": "deferred"}], "criteria": ["audience is mostly users", "ordering implies priority", "how a manager reads the project-scaling theme", "invites contribution", "non-normative examples"], "participants": {"count": 7, "roles": ["lang team members", "compiler contributors"], "decider": "team"}, "process": {"research": "none", "spike": "none", "poll": false, "escalation": "none", "subQuestions": ["scope of the roadmap (lang only or wider)", "which active initiatives are missing", "wording on moving ecosystem code into std", "why async is singled out"], "deferred": true, "reopened": false, "superseded": false, "durationDays": 0, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["the author asked whether a consensus on purpose had been reached and a participant agreed it should be stated explicitly", "the ordering debate used a reader-simulation argument about how a company manager would read the last theme", "the published document differs from the draft in theme order and adds a per-theme how-you-can-help section"]}, "outcome": {"decided": true, "decision": "A roadmap was published with three themes under a scale-empowerment frame, learning-curve theme first, project scaling last, and per-theme how-you-can-help sections. The meeting itself agreed only on purpose and on the need for clearer next steps.", "how": "unclear", "rationale": "The audience is mostly users, so user-facing themes lead; the project-scaling theme is justified as a prerequisite and invites help.", "dissent": null, "confidence": "implied", "reversibility": "stated-easy", "implementedIn": []}, "chopinFit": {"shape": "explore", "shapeWhy": "Open-ended discussion of purpose and framing for a document, with ordering as a sub-decision.", "secondaryShape": "choice", "fitsFlows": ["ask-teammate", "settle", "reshape"], "gaps": ["ranked choice: Ordering of items is a decision type: neither a single choice nor a tune dial. There is no ranked-list shape.", "off-platform decision: The thing decided is a whole document revised in place across many comments. Chopin questions settle into a sentence in prose; here the answer is a diff in a later published file, and the record would have to link meeting takes to that diff.", "implicit decision: Outcome was reached after the meeting by editing; no Settle moment, no settler. 'How it moved' needs a revision history, not a timeline of stances.", "other: Reader-simulation arguments (how a manager or a new contributor would read it) are a kind of argument not tied to an option's pros and cons.", "nested follow-ups: Meta-questions came first (purpose, scope, audience) and shaped everything later; sub-questions arrived as one `##` section per person in a synchronous meeting, a live version of Ask that the async Ask flow does not match.", "other: Non-normative examples inside a normative statement: items listed without commitment still carry weight.", "implicit decision: Suggestions accepted in the final text are only inferable by comparing two versions of a file; provenance of each change to a speaker is lost."]}, "evalUse": "Document-level prioritisation with a verifiable later state: the published roadmap shows which suggestions landed.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "Full meeting file and the heads/theme headings of the published 2024 roadmap were read. Fates for the order options and the help-wanted suggestion are inferred by comparing draft and published files (published order: learning curve, user ecosystem, project scale; per-theme how-you-can-help sections present). C++ interop and anti-examples were not found in the published text, recorded as deferred rather than rejected. Participant count is distinct handles (simulacrum and Mark appear to be one person, counted once). The author's attribution of the published order is not given in the notes."} +{"id": "rust-lang-team-2023-05-return-type-notation-spelling", "caseId": null, "collector": "cloud-git", "sources": [{"url": "https://github.com/rust-lang/lang-team/blob/c45345e32e81e273aab6c805b0d3cd7b1a666045/design-meeting-minutes/2023-05-24-return-type-notation.md", "kind": "meeting-notes", "retrievedAt": "2026-10-18T05:50:00Z"}, {"url": "https://github.com/rust-lang/rfcs/blob/d2705ad64ba47e812a2b6b19d6f613fb4c14c5e0/text/3654-return-type-notation.md", "kind": "rfc", "retrievedAt": "2026-10-18T05:50:00Z"}], "repo": "rust-lang/lang-team", "domain": "language-design", "title": "Spelling for bounding the return type of a trait method (return type notation)", "question": "How should a bound name the type returned by a trait method, so async fn in traits can require Send futures: a call-like form, a path ending in return or Output, or a general type-of operator?", "questionExplicit": true, "context": "Async functions in traits need Send bounds to be expressible on the futures they return. A draft RFC proposed a call-like notation with empty parentheses; the meeting aired concerns about confusion, extensibility and a future where function arguments could be named.", "options": [{"label": "Call-like notation m(): Send", "proposedBy": "nikomatsakis and the async working group", "fate": "partly"}, {"label": "Call-like notation with a rest marker, m(..): Send", "proposedBy": "pnkfelix (as a weak preference); async WG earlier draft", "fate": "chosen"}, {"label": "Path ending in return, m::return: Send", "proposedBy": "joshtriplett (relayed)", "fate": "rejected"}, {"label": "Path ending in Output, m::Output: Send", "proposedBy": "eholk and others", "fate": "rejected"}, {"label": "A general typeof operator", "proposedBy": "existing experimental feature", "fate": "rejected"}, {"label": "Send-trait shorthand or trait transformers", "proposedBy": "nikomatsakis", "fate": "deferred"}], "criteria": ["teachable without embarrassment", "terseness", "does not look like a call that runs", "room to name argument types later", "generalises to all type positions", "implementation cost"], "participants": {"count": 8, "roles": ["lang team members", "async working group", "compiler contributor", "absent member whose views were relayed"], "decider": "team"}, "process": {"research": "links", "spike": "prototype", "poll": false, "escalation": "none", "subQuestions": ["should impl Trait arguments be listed in turbofish", "how far should the notation generalise to other type positions", "do we stabilise associated type bounds first", "need for grouping methods"], "deferred": true, "reopened": true, "superseded": false, "durationDays": null, "comments": null, "reactionsTotal": null, "decidedOffPlatform": false, "notableMoves": ["an absent member's three positions were relayed by the champion, who agreed with one and was unpersuaded on two", "a participant demoted a strong objection to weak after thinking through a variadic future", "the rest-marker spelling had been dropped for typing ergonomics before this meeting and is the form the later RFC uses", "the later RFC says a further design meeting reviewed eight spellings with a table of verdicts"]}, "outcome": {"decided": false, "decision": "No decision at this meeting. The question was framed and the champion asked for anyone to defend another option strongly. The later RFC adopts m(..) after a second meeting that compared eight spellings.", "how": "unclear", "rationale": "Participants disliked the length of the return-path form; the Output form was liked for reusing an existing concept but seen as confusable with a future's own output; the rest marker was raised as a hedge for future argument syntax. No vote was recorded.", "dissent": "A participant worried about stabilising a notation that works in only one position and leaves the language irregular.", "confidence": "none", "reversibility": "implied", "implementedIn": []}, "chopinFit": {"shape": "choice", "shapeWhy": "A closed set of spelling options with stated pros and cons.", "secondaryShape": "explore", "fitsFlows": ["ask-teammate", "reshape", "revisit-reopen"], "gaps": ["deferred: A meeting that purposely did not decide. Settled has no 'framed and narrowed, no decision yet'; the open-question disc has no sub-status for narrowing.", "delegated decider: An absent member's stances relayed second-hand by the champion. Takes are by the person who gives them; there is no 'represented by proxy'.", "other: Options were judged against hypothetical future features (arguments, variadics, other type positions). Criteria cannot express future compatibility with features that do not exist.", "reversal after shipping: Spelling reverted to an earlier variant a year later: a loop through 'dropped, then restored' with no settled state in between. Reopen assumes a prior settle.", "rejected options content: The later RFC has a table of eight options with a verdict each. A choice card shows chosen and 'Also considered' only, not per-option reasons in a table.", "off-platform decision: Canonical artifacts live in external HackMD docs and a draft RFC; the meeting is a snapshot of how they were read.", "other: A participant argued about the criteria themselves (terseness as an argument). Disagreement about which criterion counts is not representable."]}, "evalUse": "Syntax bikeshed that ended without a decision, with relayed absent positions and a later reversal visible in an accepted RFC.", "fidelity": {"method": "git-file", "truncated": false}, "notes": "Meeting file read in full; RFC 3654 read for its syntax-alternatives table, rationale and unresolved sections only. The RFC is in the text directory, so it was merged, but its PR, FCP and merge date were not fetched. The fate of the call-like option is partly (concept kept, spelling changed); the identification of who preferred which spelling comes from the meeting notes. The prototype was a minimal implementation reported in the notes; the doc says only the associated-type-bound form was implemented at the time. participants.count counts eight attendees plus the absent member separately in roles."} diff --git a/evals/datasets/decisions-v1/documents/fetch_docs.py b/evals/datasets/decisions-v1/documents/fetch_docs.py new file mode 100644 index 000000000..9484bf98c --- /dev/null +++ b/evals/datasets/decisions-v1/documents/fetch_docs.py @@ -0,0 +1,70 @@ +"""Fetch the source documents behind documents/decisions.jsonl into the raw cache. + +Usage: python3 -I fetch_docs.py (reads every sources[].url in documents/decisions.jsonl) + +Each document is cached once as .cache/decisions/docs/.json. The +cache is git-ignored; documents/validate.py uses it for the verbatim-leak check. +""" + +import base64 +import hashlib +import json +import pathlib +import re +import subprocess +import sys +from datetime import datetime, timezone + +HERE = pathlib.Path(__file__).resolve().parent +ROOT = HERE.parents[3] +DOCS = ROOT / ".cache" / "decisions" / "docs" + + +def doc_path(url): + return DOCS / f"{hashlib.sha256(url.encode()).hexdigest()[:24]}.json" + + +def locate(url): + """Map a source URL to (owner, repo, path, ref) in a GitHub repository.""" + url = url.split("#")[0] + m = re.match(r"https://github\.com/([^/]+)/([^/]+)/blob/([^/]+)/(.+)", url) + if m: + owner, repo, ref, path = m.groups() + return owner, repo, path, ref + m = re.match(r"https://raw\.githubusercontent\.com/([^/]+)/([^/]+)/([^/]+)/(.+)", url) + if m: + owner, repo, ref, path = m.groups() + return owner, repo, path, ref + m = re.match(r"https://peps\.python\.org/pep-(\d+)/?", url) + if m: + return "python", "peps", f"peps/pep-{int(m.group(1)):04d}.rst", None + raise ValueError(f"Unsupported document URL: {url}") + + +def fetch(url): + path = doc_path(url) + if path.exists(): + return "cached" + owner, repo, file, ref = locate(url) + api = f"repos/{owner}/{repo}/contents/{file}" + (f"?ref={ref}" if ref else "") + out = subprocess.run(["gh", "api", api], capture_output=True, text=True) + if out.returncode != 0: + raise RuntimeError(out.stderr.strip()[:200]) + body = json.loads(out.stdout) + text = base64.b64decode(body["content"]).decode("utf-8", "replace") + DOCS.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"url": url, "api": api, "sha": body.get("sha"), + "retrievedAt": datetime.now(timezone.utc).isoformat(), "text": text})) + return "fetched" + + +if __name__ == "__main__": + records = [json.loads(l) for l in (HERE / "decisions.jsonl").read_text().splitlines() if l.strip()] + failed = 0 + for url in sorted({s["url"] for r in records for s in r["sources"]}): + try: + print(f"{fetch(url)}\t{url}") + except Exception as error: # report every failure and keep going + failed += 1 + print(f"FAILED\t{url}\t{error}", file=sys.stderr) + sys.exit(1 if failed else 0) diff --git a/evals/datasets/decisions-v1/documents/validate.py b/evals/datasets/decisions-v1/documents/validate.py new file mode 100644 index 000000000..da5c52cb9 --- /dev/null +++ b/evals/datasets/decisions-v1/documents/validate.py @@ -0,0 +1,103 @@ +"""Validate documents/decisions.jsonl with the main validator's record checks. + +Usage: python3 -I evals/datasets/decisions-v1/documents/validate.py + +Document records differ from thread records in four allowed ways: `caseId` is +null (there is no case file), `collector` is "cloud-git", `fidelity.method` is +"git-file", and counts a document cannot supply (`participants.count`, +`process.durationDays`, `process.comments`, `process.reactionsTotal`) may be +null. Everything else uses validate.py's checks unchanged. The leak check +compares free text with the cached source documents (fetch_docs.py). +""" + +import importlib.util +import json +import pathlib +import re +import sys + +HERE = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +from fetch_docs import doc_path # noqa: E402 + +# This file shares the main validator's name, so load that one by path. +_spec = importlib.util.spec_from_file_location("thread_validate", HERE.parent / "validate.py") +base = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(base) + +NULLABLE = ("durationDays", "comments", "reactionsTotal") + + +def check_records(records, problems): + enums = dict(base.ENUMS, collector={"cloud-git"}, **{"fidelity.method": {"git-file"}}) + saved, base.ENUMS = base.ENUMS, enums + try: + found = [] + base.check_records(records, {None: None}, found) + finally: + base.ENUMS = saved + for problem in found: + m = re.match(r"decisions\.jsonl:(\d+) \(.*\): process\.(\w+) must be an integer", problem) + if m and m.group(2) in NULLABLE and records[int(m.group(1)) - 1]["process"][m.group(2)] is None: + continue + problems.append(problem.replace("decisions.jsonl", "documents/decisions.jsonl")) + for n, r in enumerate(records, 1): + where = f"documents/decisions.jsonl:{n} ({r.get('id')})" + if r.get("caseId") is not None: + problems.append(f"{where}: caseId must be null for a document record") + count = r["participants"].get("count") + if count is not None and not isinstance(count, int): + problems.append(f"{where}: participants.count must be an integer or null") + for gap in r["chopinFit"]["gaps"]: + if not re.match(r"^[a-z][a-z -]*: \S", gap): + problems.append(f"{where}: gap is not 'cluster: detail': {gap[:60]!r}") + + +def check_leaks(records, problems): + for r in records: + source = set() + missing = [] + for s in r["sources"]: + path = doc_path(s["url"]) + if path.exists(): + source |= base.ngrams(json.loads(path.read_text())["text"]) + else: + missing.append(s["url"]) + if missing: + print(f"WARNING: {r['id']}: {len(missing)} source(s) not cached; run fetch_docs.py", file=sys.stderr) + outcome = r.get("outcome") or {} + fields = [r.get("question"), r.get("context"), r.get("title"), outcome.get("decision"), + outcome.get("rationale"), outcome.get("dissent"), r.get("notes"), r.get("evalUse"), + r["chopinFit"].get("shapeWhy")] + fields += [o.get("label") for o in r.get("options", [])] + list(r.get("criteria") or []) + fields += r["chopinFit"]["gaps"] + list(r["process"].get("notableMoves") or []) + fields += list(r["process"].get("subQuestions") or []) + for text in fields: + if isinstance(text, str): + hits = base.ngrams(text) & source + if hits: + sample = " ".join(sorted(hits)[0]) + problems.append(f"LEAK documents/decisions.jsonl {r['id']}: {len(hits)} verbatim 8-word run(s), e.g. \"{sample}\"") + + +def main(): + records = [json.loads(l) for l in (HERE / "decisions.jsonl").read_text().splitlines() if l.strip()] + threads = [json.loads(l) for l in (HERE.parent / "decisions.jsonl").read_text().splitlines() if l.strip()] + problems = [] + check_records(records, problems) + clash = {r["id"] for r in records} & {r["id"] for r in threads} + if clash: + problems.append(f"ids shared with thread records: {sorted(clash)}") + if any(doc_path(s["url"]).exists() for r in records for s in r["sources"]): + if "--no-leaks" not in sys.argv: + check_leaks(records, problems) + else: + print("WARNING: document cache absent; verbatim leak check skipped", file=sys.stderr) + for problem in problems: + print(problem) + print(f"{len(records)} document records, {len(problems)} problems") + sys.exit(1 if problems else 0) + + +if __name__ == "__main__": + main() From b2fbf4c51395ea03b9885c5e8d834f39e02add7e Mon Sep 17 00:00:00 2001 From: Maggie Appleton <5599295+MaggieAppleton@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:18:09 +0100 Subject: [PATCH 2/3] Validate document records through the shared checks documents/validate.py now passes its allowed enums, nullable counts and case-file rule to validate.check_records instead of patching its globals and filtering error text, and reuses the shared leak-field list. bun test runs the document validator alongside the thread validator. Co-Authored-By: Claude Opus 5.5 --- evals/datasets/decisions-v1.test.ts | 6 ++++ .../decisions-v1/documents/validate.py | 35 +++++-------------- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/evals/datasets/decisions-v1.test.ts b/evals/datasets/decisions-v1.test.ts index 4f3fde1b6..421c6244f 100644 --- a/evals/datasets/decisions-v1.test.ts +++ b/evals/datasets/decisions-v1.test.ts @@ -19,6 +19,12 @@ describe("decisions-v1 corpus", () => { expect(result.code).toBe(0); }); + test("passes the document-record validator", () => { + let result = validate("documents/validate.py"); + expect(result.output).toContain(" 0 problems"); + expect(result.code).toBe(0); + }); + test("cuts every case at its checkpoints the way v2 does", async () => { expect(manifest.cases.length).toBeGreaterThan(0); for (let entry of manifest.cases) { diff --git a/evals/datasets/decisions-v1/documents/validate.py b/evals/datasets/decisions-v1/documents/validate.py index da5c52cb9..6a368d74d 100644 --- a/evals/datasets/decisions-v1/documents/validate.py +++ b/evals/datasets/decisions-v1/documents/validate.py @@ -26,25 +26,14 @@ _spec.loader.exec_module(base) NULLABLE = ("durationDays", "comments", "reactionsTotal") +ENUMS = dict(base.ENUMS, collector={"cloud-git"}, **{"fidelity.method": {"git-file"}}) def check_records(records, problems): - enums = dict(base.ENUMS, collector={"cloud-git"}, **{"fidelity.method": {"git-file"}}) - saved, base.ENUMS = base.ENUMS, enums - try: - found = [] - base.check_records(records, {None: None}, found) - finally: - base.ENUMS = saved - for problem in found: - m = re.match(r"decisions\.jsonl:(\d+) \(.*\): process\.(\w+) must be an integer", problem) - if m and m.group(2) in NULLABLE and records[int(m.group(1)) - 1]["process"][m.group(2)] is None: - continue - problems.append(problem.replace("decisions.jsonl", "documents/decisions.jsonl")) + base.check_records(records, {}, problems, enums=ENUMS, nullable=NULLABLE, + file="documents/decisions.jsonl", require_case=False) for n, r in enumerate(records, 1): where = f"documents/decisions.jsonl:{n} ({r.get('id')})" - if r.get("caseId") is not None: - problems.append(f"{where}: caseId must be null for a document record") count = r["participants"].get("count") if count is not None and not isinstance(count, int): problems.append(f"{where}: participants.count must be an integer or null") @@ -65,19 +54,11 @@ def check_leaks(records, problems): missing.append(s["url"]) if missing: print(f"WARNING: {r['id']}: {len(missing)} source(s) not cached; run fetch_docs.py", file=sys.stderr) - outcome = r.get("outcome") or {} - fields = [r.get("question"), r.get("context"), r.get("title"), outcome.get("decision"), - outcome.get("rationale"), outcome.get("dissent"), r.get("notes"), r.get("evalUse"), - r["chopinFit"].get("shapeWhy")] - fields += [o.get("label") for o in r.get("options", [])] + list(r.get("criteria") or []) - fields += r["chopinFit"]["gaps"] + list(r["process"].get("notableMoves") or []) - fields += list(r["process"].get("subQuestions") or []) - for text in fields: - if isinstance(text, str): - hits = base.ngrams(text) & source - if hits: - sample = " ".join(sorted(hits)[0]) - problems.append(f"LEAK documents/decisions.jsonl {r['id']}: {len(hits)} verbatim 8-word run(s), e.g. \"{sample}\"") + for text in base.record_texts(r): + hits = base.ngrams(text) & source + if hits: + sample = " ".join(sorted(hits)[0]) + problems.append(f"LEAK documents/decisions.jsonl {r['id']}: {len(hits)} verbatim 8-word run(s), e.g. \"{sample}\"") def main(): From 25863247c645020d03d5fa13b4082edc97451ad3 Mon Sep 17 00:00:00 2001 From: Maggie Appleton <5599295+MaggieAppleton@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:28:45 +0100 Subject: [PATCH 3/3] Note that the holdout check covered the document records Co-Authored-By: Claude Opus 5.5 --- evals/datasets/decisions-v1/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/datasets/decisions-v1/README.md b/evals/datasets/decisions-v1/README.md index 660770d21..a935b5860 100644 --- a/evals/datasets/decisions-v1/README.md +++ b/evals/datasets/decisions-v1/README.md @@ -71,7 +71,7 @@ Of the 192 candidates: ## Holdout check On 2026-10-19 every candidate, case source, related thread, decision-record -source and implementation link was checked against the 8 holdout threads: 5 for +source and implementation link, including the 41 document records, was checked against the 8 holdout threads: 5 for validation from eval-hub partition snapshot `dda78480888859ae3eec90846c1263cd28552d05`, and 3 sealed for final tests. None matched, and no kept thread or source document links to any of them. Every