diff --git a/bin/check-post-voice b/bin/check-post-voice new file mode 100755 index 000000000..fc6ff2458 --- /dev/null +++ b/bin/check-post-voice @@ -0,0 +1,124 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true +# +# Mechanical check for blog-pipeline.md STEP 4b (cadence quotas) and STEP 5a +# (anti-AI tells). Both were prose checklists, and prose checklists get skipped: +# three posts shipped on 2026-08-22 having passed neither, and the slop was +# visible to a reader immediately. +# +# bin/check-post-voice content/blog//index.md +# +# Exits non-zero if any rule fails, so it can gate a commit. +# It measures SHAPE only. It cannot tell you whether the post is any good. + +require "set" + +path = ARGV[0] or abort "usage: bin/check-post-voice " +raw = File.read(path, encoding: "bom|utf-8") +parts = raw.split(/^---\s*$/m) +body = parts[2].to_s + +MANNERISM = %w[discipline cycle pattern canonical workflow framework cadence + rhythm contract gate gatekeeper layer round review pass].freeze +BANNED = %w[unlock harness leverage game-changer journey seamless robust + supercharge revolutionize embark delve discover].freeze + +# --- split into sections of prose paragraphs, skipping fenced code ------------ +sections = Hash.new { |h, k| h[k] = [] } +current = "(lead)" +buf = [] +in_code = false + +flush = lambda do + para = buf.join(" ").strip + buf = [] + return if para.empty? + sections[current] << para +end + +body.each_line do |line| + if line.strip.start_with?("```") + flush.call unless in_code + in_code = !in_code + next + end + next if in_code + + if line =~ /^##+\s+(.*)/ + flush.call + current = Regexp.last_match(1).strip + elsif line.strip.empty? + flush.call + elsif !line.start_with?("#", ">", "-", "|", "!") + buf << line.strip + end +end +flush.call + +failures = [] +prose = sections.values.flatten +abort "no prose found in #{path}" if prose.empty? + +# --- STEP 4b: cadence quotas, per H2 ----------------------------------------- +sections.each do |name, paras| + sents = paras.flat_map { |p| p.split(/(?<=[.!?])\s+/) }.reject { |s| s.strip.empty? } + next if sents.empty? + lens = sents.map { |s| s.split.size } + + failures << "#{name}: no sentence under 9 words (min #{lens.min})" if lens.min >= 9 + failures << "#{name}: no sentence over 22 words (max #{lens.max})" if lens.max <= 22 + failures << "#{name}: no single-sentence paragraph" unless paras.any? { |p| p.split(/(?<=[.!?])\s+/).reject { |x| x.strip.empty? }.size == 1 } + + over = paras.count { |p| p.split(/(?<=[.!?])\s+/).reject { |x| x.strip.empty? }.size > 3 } + failures << "#{name}: #{over} paragraph(s) over the 3-sentence cap" if over.positive? +end + +# --- STEP 5a: anti-AI tells -------------------------------------------------- +sents = prose.flat_map { |p| p.split(/(?<=[.!?])\s+/) }.reject { |s| s.strip.empty? } + +tell = lambda do |label, matches| + return if matches.empty? + failures << "#{label} (#{matches.size}): #{matches.first.strip[0, 90]}" +end + +tell.("negative parallelism", sents.select { |s| s.strip =~ /\ANot [a-z]/ }) +# Only flags the generic subject DOING something ("the team ships", "most +# founders assume"). "anyone on the team could remember" names a real team and +# is not the tell - matching it was a false positive on 2026-08-22. +tell.("generalized actor", sents.select { |s| s =~ /\b(the developer|the team|most teams|most founders)\s+(is|are|was|were|has|have|will|can|should|does|do|needs?|assumes?|ships?|writes?)\b/i }) +tell.("definitional cadence", sents.select { |s| s =~ /\b(is|that is) the (canonical|largest|whole|real|part|version|move|rule|reason|kind)\b/i }) +tell.("signposting", sents.select { |s| s =~ /\A(This post|Here'?s what|In this article|Let'?s (dive|explore))/i }) +tell.("therapist voice", sents.select { |s| s =~ /that'?s the (hardest|worst|tricky)/i }) +tell.("copula avoidance", sents.select { |s| s =~ /\bserves as\b/i }) + +banned_hits = BANNED.select { |w| body.downcase =~ /\b#{Regexp.escape(w)}\b/ } +failures << "banned words: #{banned_hits.join(', ')}" unless banned_hits.empty? +failures << "em dash present - use '-'" if body.include?("—") + +mann = MANNERISM.sum { |w| body.scan(/\b#{w}s?\b/i).size } +failures << "mannerism nouns: #{mann} (cap 5)" if mann > 5 + +opener = prose.count { |p| p =~ /\A(The|We)\b/ } +pct = (100.0 * opener / prose.size).round +failures << "The/We paragraph openers: #{pct}% (cap 20%)" if pct > 20 + +# --- artifact + receipts (2026-08-22: every post owes both) ------------------ +code_blocks = body.scan(/^```/).size / 2 +images = body.scan(/!\[[^\]]*\]\(/).size +failures << "no artifact: 0 code blocks and 0 images" if code_blocks.zero? && images.zero? + +external = body.scan(%r{\]\((https?://[^)]+)\)}).flatten.reject { |u| u.include?("jetthoughts.com") } +failures << "no receipts: zero external citations" if external.empty? + +# --- report ------------------------------------------------------------------ +puts "#{path}" +puts " #{prose.size} paragraphs, #{code_blocks} code blocks, #{images} images, #{external.size} external citations" + +if failures.empty? + puts " PASS" + exit 0 +end + +puts " FAIL (#{failures.size})" +failures.each { |f| puts " - #{f}" } +exit 1 diff --git a/content/blog/how-to-audit-content-you-didnt-write/domains.svg b/content/blog/how-to-audit-content-you-didnt-write/domains.svg new file mode 100644 index 000000000..d6e6ffbaa --- /dev/null +++ b/content/blog/how-to-audit-content-you-didnt-write/domains.svg @@ -0,0 +1,30 @@ + + Share of pages showing AI-authorship signals, by domain type + Pew Research, July 2026: about 10 percent of .com pages show significant signs of AI authorship, against 4.6 percent for .org and roughly 1 percent for .edu and .gov. Sample of about 490,000 Common Crawl pages analysed with Open Pangram. + + + + Where AI-written pages concentrate + Pew Research, July 2026 - ~490,000 Common Crawl pages, Open Pangram + + + .com + + + ~10% + + + .org + + + 4.6% + + + .edu + + + ~1% + + Commercial sites carry ten times what universities do. + Yours is a commercial site. + diff --git a/content/blog/how-to-audit-content-you-didnt-write/index.md b/content/blog/how-to-audit-content-you-didnt-write/index.md index 2e82def01..33c4ae44f 100644 --- a/content/blog/how-to-audit-content-you-didnt-write/index.md +++ b/content/blog/how-to-audit-content-you-didnt-write/index.md @@ -38,17 +38,15 @@ Then a tool started drafting, and the person approving its output was not equipp ## Nobody can tell you how much of the web this is -You would think there is a number. There are several and they disagree. +You would think there is a number. There are several and they disagree - ten percent, a third, or half, depending on whose sample and whose detector. -[Graphite](https://graphite.io/five-percent/more-articles-are-now-created-by-ai-than-humans) sampled 43,000 CommonCrawl URLs published between January 2020 and May 2025, ran them through Surfer's detector, and put the crossover - more machine-written articles than human ones - in November 2024. +[Graphite](https://graphite.io/five-percent/more-articles-are-now-created-by-ai-than-humans) put the crossover, more machine-written articles than human ones, in November 2024. [Pew](https://www.pewresearch.org/data-labs/2026/08/20/how-much-of-the-internet-is-written-with-ai/) ran ~490,000 Common Crawl pages through Open Pangram this month and found 10% carrying AI-authorship signals, rising to over a third among pages published after ChatGPT shipped. Both publish their error bars, which is the habit worth stealing whatever you make of the figures. -[Pew](https://www.pewresearch.org/data-labs/2026/08/20/how-much-of-the-internet-is-written-with-ai/) ran roughly 490,000 English-language pages from the same archive through Open Pangram this month and found 10% of all sampled pages showing significant signs of AI authorship - rising to over a third when you look only at pages published after ChatGPT shipped. +Pew also split it by domain, and that is where you come in: -Ten percent, a third, or half. The numbers are not contradicting each other so much as answering different questions, on different samples, with different detectors. +![Share of pages showing AI-authorship signals by domain: .com around 10 percent, .org 4.6 percent, .edu and .gov around 1 percent.](domains.svg) -Both teams publish their error bars, and that habit is worth stealing regardless of what you make of their numbers. Graphite reports a 4.2% false-positive rate against GPT-4o content and no measurement at all of AI-assisted-then-human-edited writing. Pew says plainly that "AI detection models aren't perfect" on individual documents and only hold up in aggregate. - -One more number from Pew is the one that should interest you: **around one in ten `.com` pages carry those signals - roughly double the `.org` rate and ten times what `.edu` and `.gov` show.** Commercial sites are where this concentrates, and yours is a commercial site. +Commercial sites are where this concentrates, and yours is a commercial site. So the honest position is that the web's average is unknown and the detectors that estimate it are themselves approximate. Which is fine, because the average was never the thing you needed. **You need to know about your property, and your property is countable.** @@ -95,7 +93,7 @@ Run the first one and read every hit. Real client work names the client or does Count how many of your long posts link to nothing outside your own domain. -A post with no external citation is not necessarily wrong. It is unverifiable, which means nobody could have checked it - including whoever wrote it, at the moment they wrote it. That is the distinction I care about, and it is the one this count actually measures. +A post with no external citation is not necessarily wrong. It is unverifiable, which means nobody could have checked it - including whoever wrote it, at the moment they wrote it. Uncheckable is where wrong survives, and the number usually comes back higher than anyone guesses. @@ -131,9 +129,9 @@ Then check each against the vendor's own support table. Laravel, Rails and Node It cannot tell you whether a claim is true. -We tried to build that and failed honestly. Two candidate patterns for detecting wrong technical claims got measured against the archive before we trusted either: one matched ten passages, mostly legitimate; the other matched a hundred and eighty-five, almost all ordinary prose. Both would have cried wolf until people stopped reading the output. +We tried to build that and failed. Two candidate patterns for detecting wrong technical claims got measured before either was trusted: one matched ten passages, mostly legitimate, the other a hundred and eighty-five, almost all ordinary prose. Both would have cried wolf until people stopped reading the output. -A wrong explanation is a well-formed sentence using correct vocabulary, pointing roughly the right direction. There is no pattern for it. What catches it is someone who knows the subject, reading with permission to disagree - which is a different investment, and the one worth making after the four checks above have narrowed where to look. +A wrong explanation is a well-formed sentence using correct vocabulary, pointing roughly the right direction. What catches it is someone who knows the subject reading with permission to disagree - a different investment, and the one worth making after these four have narrowed where to look. ## Where to start this week diff --git a/content/blog/what-senior-developers-catch-that-ai-misses/chain.svg b/content/blog/what-senior-developers-catch-that-ai-misses/chain.svg new file mode 100644 index 000000000..ee27bf59c --- /dev/null +++ b/content/blog/what-senior-developers-catch-that-ai-misses/chain.svg @@ -0,0 +1,36 @@ + + Three links in the review chain, only one of which can be automated + A model writes, a second model challenges, and a person with domain knowledge adjudicates. The first two are automatable; the third is not. Drop the third and two confident systems agree with each other. + + + + Three links. Only one of them is a person. + + + + 1 - AUTOMATED + A model writes + fluent, confident, fast + + + + + + + 2 - AUTOMATED + Another challenges + different type, told to object + + + + + + + 3 - NOT AUTOMATED + A person referees + knows the answer already + + Drop link 3 + and you have two confident systems agreeing with each other. + That agreement looks exactly like a passing review. + diff --git a/content/blog/what-senior-developers-catch-that-ai-misses/index.md b/content/blog/what-senior-developers-catch-that-ai-misses/index.md index cb8679337..1c02911dd 100644 --- a/content/blog/what-senior-developers-catch-that-ai-misses/index.md +++ b/content/blog/what-senior-developers-catch-that-ai-misses/index.md @@ -16,9 +16,7 @@ canonical_url: 'https://jetthoughts.com/blog/what-senior-developers-catch-that-a related_posts: false --- -Cards on the table before I start: I run a development shop, and this post argues you need experienced people reviewing AI output. That is convenient for me. So I am going to make the case with a diff you can check yourself, and if the Rails reasoning does not hold up, none of the rest should persuade you either. - -Here is a change we stopped in review. It is small, it is plausible, and it is wrong in a way you cannot see without knowing Rails. +Here is a change we caught before it merged. It is small, it is plausible, and it is wrong in a way you cannot see without knowing Rails. ```diff - Propshaft is dramatically faster than Sprockets: precompilation drops from @@ -29,7 +27,9 @@ Here is a change we stopped in review. It is small, it is plausible, and it is w The deletion is correct. That timing figure had no measurement behind it and deserved to go. -Read the addition again. It is wrong. +Read the addition again. + +It is wrong. Propshaft still walks every asset, fingerprints it, and copies it into place. Its own README says so: @@ -41,31 +41,15 @@ So a made-up number was swapped for a made-up mechanism. That is the worse trade ## Nobody skimming that paragraph would have stopped -Sit with that for a second. - The sentence had a subject, a cause and an effect, and it used the right vocabulary throughout. It also agreed with the general direction of the truth - Propshaft *is* faster - while getting the reason for it wrong. You cannot catch that by reading carefully. -You catch it by already knowing what `assets:precompile` does. - -Hacker News spent the last three weeks arguing about exactly this, mostly without noticing it was one argument. Sean Goedecke's ["LLMs reward expertise"](https://www.seangoedecke.com/llms-reward-expertise/) drew 573 comments with a simple claim: "The most important skill in prompting is expertise in the domain you're prompting for." - -Two weeks later Senko Rašić's ["'Code was never the hard part' is an insult to all programmers"](https://blog.senko.net/code-was-never-the-hard-part-is-an-insult-to-all-programmers) drew 590 more, insisting that "creating good code is a craft that requires skill, patience, attention to detail, experience and wisdom." - -Both threads circle the same question and neither settles it. If the model writes the code, what is the person for? - -## What the diff answers - -It was not the typing. +You catch it by already knowing what `assets:precompile` does. Sean Goedecke put the general version well in ["LLMs reward expertise"](https://www.seangoedecke.com/llms-reward-expertise/): "The most important skill in prompting is expertise in the domain you're prompting for." -The prose came out clean, grammatical and confident on the first pass, and would have survived any editor who did not happen to know how Rails compiles assets. +He calls the thing experts do "steering" - you recognise a suboptimal suggestion and redirect it. This diff is that mechanism running backwards. Without someone who knows the asset pipeline, there is nothing to steer against and the confident answer wins by default. -What it could not do was notice that its own explanation was false. It had no way to check, because checking meant knowing something about the Rails asset pipeline that was not in the sentence it had just written. - -Goedecke calls the thing experts do "steering" - you recognise a suboptimal suggestion and redirect it. This diff is that mechanism running backwards: without someone who knows the asset pipeline, there is nothing to steer against, and the confident answer wins by default. - -Note what the change was *for*. The task was removing an unsourced number, and the same edit introduced a new defect while completing it. Cleanup is where this happens most, because a correction feels like tidying rather than authorship, and gets read that way. +Note what the change was *for*. The task was removing an unsourced number, and the same edit introduced a new defect while completing it. Cleanup is where this happens most, because a correction feels like tidying rather than authorship. ## What actually caught it @@ -77,10 +61,12 @@ It came back with four findings. This was one, stated flatly: > On applications with many assets, Propshaft still enumerates, fingerprints, and copies every asset during `assets:precompile`, so its work still scales with asset count. Removing transpilation and concatenation reduces the per-asset cost but does not make the build independent of asset count; the new wording gives readers an incorrect performance expectation. -Then a person had to decide whether the reviewer was right, and I want to be precise about what that took: either knowing the answer already, or being willing to go and read the Propshaft source until you did. +Then a person had to decide whether the objection was correct, and that step took either already knowing the answer or being willing to go and read the Propshaft source until you did. Three links in that chain, and only one of them is automatable. A model wrote, another model challenged, and someone with domain knowledge adjudicated. +![Three links in the chain: a model writes, a second model challenges, a person referees. Only the first two are automatable.](chain.svg) + Drop the third link and you have two confident systems agreeing with each other. ## It is not only the model that is confident @@ -91,7 +77,7 @@ They were 19% slower with AI. Going in, those developers expected a 24% speedup. Coming out, having just lived through the slowdown, they still estimated AI had made them 20% faster. Roughly forty points between what happened and what they believed happened - in experts, on their own code. -METR is careful about what that does not show, so I will be too: early-2025 models, Claude 3.5 and 3.7 Sonnet through Cursor Pro, on mature codebases with demanding quality standards, and they explicitly decline to claim it generalises to most developers or to later tools. +METR is careful about what that does not show: early-2025 models, Claude 3.5 and 3.7 Sonnet through Cursor Pro, on mature codebases with demanding quality standards, and they explicitly decline to claim it generalises to most developers or to later tools. What travels is the gap itself. Experience did not make those developers better at estimating their own speed. It made them more certain about an estimate that was forty points off. @@ -123,16 +109,6 @@ Three things worth asking, none of which require you to read code: We wrote about the [team structure that makes this hold up](/blog/claude-code-xp-team-workflow/) if you want the operating detail. The short version fits in a sentence: the reviewer has to be someone other than the writer, and they have to be allowed to say no. -## The uncomfortable version - -Every defect in this post was caught the same way: a second pass whose brief was to disagree, followed by someone who knew the subject well enough to referee the disagreement. - -Neither half works alone. The reviewer that only agrees is decoration, and the reviewer that objects to something nobody can adjudicate is noise. - -Expertise earns its money in a handful of moments per week, and none of them look like productivity. Someone reads a paragraph that scans perfectly and says no, and cannot always explain why until they go and check. - -That is an awkward thing to sell and an awkward thing to measure. I would still rather tell you that than quote you a velocity number. - ## Sources - Sean Goedecke, ["LLMs reward expertise"](https://www.seangoedecke.com/llms-reward-expertise/) - [HN discussion](https://news.ycombinator.com/item?id=49161518), 573 comments diff --git a/content/blog/when-did-a-test-last-fail-on-purpose/checked.svg b/content/blog/when-did-a-test-last-fail-on-purpose/checked.svg new file mode 100644 index 000000000..d6a60c4b5 --- /dev/null +++ b/content/blog/when-did-a-test-last-fail-on-purpose/checked.svg @@ -0,0 +1,27 @@ + + Links actually checked, before and after adding the --remap flag + Before: 15,642 of 149,516 links checked, about one tenth. After: 114,239 checked. Both runs reported zero errors. + + + + Links the checker actually inspected + Same site. Same command. One extra flag. + + + BEFORE + + + 15,642 checked + 133,874 skipped + + + AFTER + + + 114,239 checked + 35,501 external + + Both runs reported + 0 Errors + - only one of them meant it. + diff --git a/content/blog/when-did-a-test-last-fail-on-purpose/index.md b/content/blog/when-did-a-test-last-fail-on-purpose/index.md index e54bc724d..21d62bb18 100644 --- a/content/blog/when-did-a-test-last-fail-on-purpose/index.md +++ b/content/blog/when-did-a-test-last-fail-on-purpose/index.md @@ -58,6 +58,8 @@ Same command, same flags, one addition. The next run: From 15,642 links checked to 114,239. +![Links actually inspected before and after adding --remap: 15,642 of 149,516 versus 114,239. Both runs reported zero errors.](checked.svg) + ## Run this on your own repo before you keep reading The diagnostic is cheap. @@ -110,7 +112,7 @@ def test_rendered_pages_do_not_regress_on_banned_phrases end ``` -Setting it to 10 and watching it fail takes fifteen seconds. As far as I can tell it is the only thing separating a ratchet from a decoration. +Setting it to 10 and watching it fail takes fifteen seconds. It is the only thing separating a ratchet from a decoration. ## The one that should genuinely worry you @@ -149,7 +151,7 @@ A new test is not finished until someone has broken the thing it guards and watc That adds maybe two minutes to writing a test. -I think it is the highest-return two minutes in the whole suite, and I would rather tell you that than quote you a coverage percentage that cannot tell a working check apart from a decorative one. +It is the highest-return two minutes in the suite, and worth more than a coverage percentage that cannot tell a working check apart from a decorative one. If you want the exercise: pick your three most important checks, plant one realistic defect against each, and write down beforehand which one should catch it. You will learn more in an afternoon than a coverage report has told you all year. diff --git a/docs/workflows/blog-pipeline.md b/docs/workflows/blog-pipeline.md index a954e04d7..66eaffa7b 100644 --- a/docs/workflows/blog-pipeline.md +++ b/docs/workflows/blog-pipeline.md @@ -240,7 +240,19 @@ way. Two candidate markers were measured and rejected (10 and 185 hits, both mostly legitimate prose) — see `.okf/content/fabrication-ratchet.md`. STEP 5a — ANTI-AI WRITING PASS (MANDATORY — run BEFORE review loop) -Run /humanizer on the draft. Scan every paragraph for these AI tells: + +RUN THE SCRIPT FIRST — `bin/check-post-voice content/blog//index.md`. +It measures STEP 4b's cadence quotas and the tell list below mechanically, and +exits non-zero on failure. This exists because the prose checklist kept getting +skipped: three posts shipped on 2026-08-22 having passed neither, and the slop +was obvious to a reader immediately. It also enforces the two rules that are not +cadence choices — an artifact (code block or image) and at least one external +citation. + +It measures SHAPE. It cannot tell you the post is good, so the human passes +below still run — but do not send a draft to them that the script rejects. + +Then run /humanizer on the draft. Scan every paragraph for these AI tells: - Rule of three (parallel triads) — break or combine - Signposting ("This post is about...", "Here's what you need to know") - Bold inline-header lists — rewrite as flowing prose