From 494a3447005e7261caa25efec3a05a3d740a9f20 Mon Sep 17 00:00:00 2001 From: Kjell Tore Guttormsen Date: Fri, 19 Jun 2026 13:52:30 +0200 Subject: [PATCH] =?UTF-8?q?docs(voyage):=20S19=20=E2=80=94=20leanness:=20h?= =?UTF-8?q?onest=20test=20census=20+=20prune=203=20redundant=20prose-pins?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes devils-advocate audit §Top changes #8 (MINOR×MED). Reports the behavior-test count separately from the doc-consistency prose-pin count so the cited total no longer oversells behavior coverage, and removes the clearest reword-fragile redundancies. Conservative scope (operator-chosen): no genuine regression guard dropped. 1. Suite census (report separately). lib/util/test-census.mjs + tests/lib/test-census.test.mjs walk tests/**/*.test.mjs and bucket top-level test() declarations into behavior vs doc-consistency-pins (bucket per file, regex PIN_FILE_RE), asserting the two sum to the total so neither can drift silently. Split emitted as a t.diagnostic: behavior=601 doc-consistency-pins=69. Metric = top-level declarations; node:test's runtime total counts subtests too and is therefore >= it. 2. Conservative prune (3 tests, each provably subsumed): - "trekexecute.md still parses v1.7 plan schema" — tautological OR-chain; real coverage = the plan_version:1.7 template pin + plan-validator / plan-schema behavior tests. - "CLAUDE.md mentions all six pipeline commands" — hardcoded six-string list subsumed by the filesystem-driven "commands table mentions every commands/*.md file" structural pin. - "CLAUDE.md mentions /trekcontinue command" — same subsumption. Each removal leaves an in-place note (why + where coverage lives). 3. doc-consistency.test.mjs header now documents the structural-invariant vs prose/existence-pin distinction so new pins land in the right kind. TDD: census test written failing-first (ERR_MODULE_NOT_FOUND before lib/util/test-census.mjs existed). Suite 699->698 (696 pass / 2 skip / 0 fail): -3 prune +2 census. plugin validate passes (1 accepted warning). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01LqBYc8Ltrk7LipyJmGxXiB --- CHANGELOG.md | 1 + lib/util/test-census.mjs | 49 ++++++++++++++++++++++++ tests/lib/doc-consistency.test.mjs | 61 ++++++++++++++---------------- tests/lib/test-census.test.mjs | 43 +++++++++++++++++++++ 4 files changed, 121 insertions(+), 33 deletions(-) create mode 100644 lib/util/test-census.mjs create mode 100644 tests/lib/test-census.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 28cc703..0e79a70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ Existing `2.0`/`2.1` briefs stay valid (forward + backward compatible), mirrorin ### Release hygiene - **Version sync** — `plugin.json`, `package.json`, README badge, and the CHANGELOG top entry all at `5.5.0`, guarded by a new version-consistency test in `tests/lib/doc-consistency.test.mjs`. +- **Honest test count (S19)** — `lib/util/test-census.mjs` + `tests/lib/test-census.test.mjs` report the behavior-test count separately from the doc-consistency prose-pin count, so the cited total no longer oversells behavior coverage. Pruned 3 redundant/tautological prose-pins from `doc-consistency.test.mjs` (each provably subsumed by a structural invariant or behavior test). Devil's-advocate audit §Top changes #8. - **`claude plugin validate` passes.** The single advisory warning (root `CLAUDE.md` not loaded as consumer project context) is **accepted by design**: it is universal across all marketplace plugins, validation still passes, and the root `CLAUDE.md` is repo/maintainer context — consumer context ships via README + command/agent frontmatter. ## v5.1.1 — 2026-05-14 — Remediation patch (11/12 review findings closed) diff --git a/lib/util/test-census.mjs b/lib/util/test-census.mjs new file mode 100644 index 0000000..1281e38 --- /dev/null +++ b/lib/util/test-census.mjs @@ -0,0 +1,49 @@ +// lib/util/test-census.mjs +// Census of the test suite: split top-level test() declarations into +// "behavior" tests vs "doc-consistency pins" (string/existence assertions that +// pin documentation against a source-of-truth). Makes the cited test count +// honest — a prose-pin is not the same coverage as a behavior test, so a single +// conflated total oversells behavior coverage. +// Devil's-advocate audit §Top changes #8 (S19). +// +// Metric: top-level `test(` declarations (static, deterministic, in-process). +// This is distinct from node:test's runtime total, which additionally counts +// subtests; the runtime total is therefore ≥ this declaration count. + +import { readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; + +// Files whose tests are documentation pins, not behavior coverage. Kept as a +// regex (not a single filename) so a future split-out (e.g. prose-pins.test.mjs) +// is bucketed correctly without editing this module. +export const PIN_FILE_RE = /(doc-consistency|prose-pins)\.test\.mjs$/; + +function walk(dir) { + const out = []; + for (const e of readdirSync(dir, { withFileTypes: true })) { + const p = join(dir, e.name); + if (e.isDirectory()) out.push(...walk(p)); + else if (e.isFile() && e.name.endsWith('.test.mjs')) out.push(p); + } + return out; +} + +function countTests(file) { + return (readFileSync(file, 'utf-8').match(/^\s*test\(/gm) || []).length; +} + +// Returns { behavior, docPins, total, byFile } for all *.test.mjs under +// testsRoot. behavior + docPins === total by construction. +export function censusTests(testsRoot) { + const files = walk(testsRoot).sort(); + const byFile = {}; + let behavior = 0; + let docPins = 0; + for (const f of files) { + const n = countTests(f); + byFile[f] = n; + if (PIN_FILE_RE.test(f)) docPins += n; + else behavior += n; + } + return { behavior, docPins, total: behavior + docPins, byFile }; +} diff --git a/tests/lib/doc-consistency.test.mjs b/tests/lib/doc-consistency.test.mjs index c6374b1..9f45b1b 100644 --- a/tests/lib/doc-consistency.test.mjs +++ b/tests/lib/doc-consistency.test.mjs @@ -4,6 +4,19 @@ // // When this test fails, fix the source-of-truth — do NOT rewrite the test to // hide drift. Borrowed pattern from llm-security commit 97c5c9d. +// +// Two kinds of pin live here (devil's-advocate audit §Top changes #8, S19): +// • STRUCTURAL cross-file invariants — derive one side from a source-of-truth +// (file counts, dimension counts from `### N.` headers, profile tables from +// the yaml, version-sync) and survive harmless rewording. These are the +// load-bearing ones; keep and grow them. +// • PROSE / EXISTENCE pins — assert a literal string or file fact. Most still +// guard a real wiring (CLI shims, annotate.mjs, anti-stale CHANGELOG, +// anti-false-claim), but they break on reword. Prefer a structural form +// when one exists; do not add a prose-pin that a structural test subsumes. +// All tests in this file count as "doc-consistency pins" in the suite census +// (lib/util/test-census.mjs) so the cited test total stays honest about how much +// is behavior coverage vs documentation pinning. import { test } from 'node:test'; import { strict as assert } from 'node:assert'; @@ -129,16 +142,12 @@ test('templates/plan-template.md declares plan_version: 1.7', () => { assert.match(tpl, /plan_version:\s*['"]?1\.7['"]?/); }); -test('commands/trekexecute.md still parses v1.7 plan schema', () => { - const cmd = read('commands/trekexecute.md'); - const tpl = read('templates/plan-template.md'); - const tplVersion = (tpl.match(/plan_version:\s*['"]?([\d.]+)['"]?/) || [])[1]; - assert.ok(tplVersion, 'templates/plan-template.md missing plan_version'); - assert.ok( - cmd.includes(`plan_version`) || cmd.includes(`Step N:`) || cmd.includes('### Step '), - 'commands/trekexecute.md should reference v1.7 plan-schema parsing', - ); -}); +// S19 prune (audit §Top changes #8): removed a tautological prose-pin — +// "commands/trekexecute.md still parses v1.7 plan schema" was an OR-chain +// (`includes('plan_version') || includes('Step N:') || includes('### Step ')`) +// that is ~always true and asserts almost nothing. The real v1.7 coverage lives +// in the structural pin "templates/plan-template.md declares plan_version: 1.7" +// (above) + the plan-validator / plan-schema behavior tests. test('settings.json has only known top-level scopes after Spor 0 cleanup', () => { const cfg = JSON.parse(read('settings.json')); @@ -156,21 +165,11 @@ test('settings.json no longer carries vestigial exploration block', () => { 'agentTeam block was vestigial — should be deleted in v3.1.0 Spor 0'); }); -test('CLAUDE.md mentions all six pipeline commands', () => { - // v4.1 Step 21 — added /trekcontinue to coverage (was 5/6 before). - // v5.0.0 — /trekrevise removed (bespoke playground retired); back to six. - const md = read('CLAUDE.md'); - for (const c of [ - '/trekbrief', - '/trekresearch', - '/trekplan', - '/trekexecute', - '/trekreview', - '/trekcontinue', - ]) { - assert.ok(md.includes(c), `CLAUDE.md missing reference to ${c}`); - } -}); +// S19 prune (audit §Top changes #8): removed "CLAUDE.md mentions all six +// pipeline commands" — a hardcoded six-string list fully subsumed by the +// structural pin "CLAUDE.md commands table mentions every commands/*.md file" +// above, which derives the command set from the filesystem and auto-covers any +// new command. The hardcoded list was the weaker, reword-fragile form. test('HANDOVER-CONTRACTS.md contains Handover 6 section', () => { const text = read('docs/HANDOVER-CONTRACTS.md'); @@ -248,14 +247,10 @@ test('HANDOVER-CONTRACTS.md Handover 7 § Lifecycle names --cleanup and produced ); }); -test('CLAUDE.md mentions /trekcontinue command', () => { - const md = read('CLAUDE.md'); - assert.ok( - md.includes('/trekcontinue') || md.includes('trekcontinue'), - 'CLAUDE.md should document /trekcontinue in the Commands table ' + - '(added in v3.3.0 alongside the new command file)', - ); -}); +// S19 prune (audit §Top changes #8): removed "CLAUDE.md mentions /trekcontinue +// command" — subsumed by "CLAUDE.md commands table mentions every commands/*.md +// file" (commands/trekcontinue.md exists, so that loop already asserts the +// /trekcontinue reference). Single-command existence pin → redundant. test('rule-catalogue has exactly 12 entries', async () => { const mod = await import('../../lib/review/rule-catalogue.mjs'); diff --git a/tests/lib/test-census.test.mjs b/tests/lib/test-census.test.mjs new file mode 100644 index 0000000..92fb0d9 --- /dev/null +++ b/tests/lib/test-census.test.mjs @@ -0,0 +1,43 @@ +// tests/lib/test-census.test.mjs +// Behavior test for the suite census + the honest-count invariant from the +// devil's-advocate audit §Top changes #8 (S19): the behavior-test count must +// be reported SEPARATELY from the doc-consistency prose-pin count. A prose-pin +// (a string/existence assertion against documentation) is not the same coverage +// as a behavior test, so a single conflated total oversells the behavior +// coverage. This census makes the split explicit and drift-proof. + +import { test } from 'node:test'; +import { strict as assert } from 'node:assert'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { censusTests } from '../../lib/util/test-census.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const TESTS_ROOT = join(HERE, '..'); + +test('suite census splits behavior tests from doc-consistency pins (S19)', (t) => { + const c = censusTests(TESTS_ROOT); + // Honest-count invariant: the two buckets must account for every top-level + // test() declaration — no silent drift between behavior and pin counts. + assert.equal(c.behavior + c.docPins, c.total, + 'census buckets must sum to the total declaration count'); + assert.ok(c.docPins > 0, 'doc-consistency pin bucket must be non-empty (regex/glob sanity)'); + assert.ok(c.behavior > 0, 'behavior bucket must be non-empty (regex/glob sanity)'); + // Report the split so the cited count is honest (audit §Top changes #8). + // Metric = top-level test() declarations; node:test's runtime total counts + // subtests too and is therefore ≥ this number. + t.diagnostic( + `behavior=${c.behavior} doc-consistency-pins=${c.docPins} ` + + `total=${c.total} (top-level test() declarations)`, + ); +}); + +test("doc-consistency.test.mjs is the suite's prose-pin home (S19)", () => { + const c = censusTests(TESTS_ROOT); + const pinFiles = Object.keys(c.byFile) + .filter((f) => /(doc-consistency|prose-pins)\.test\.mjs$/.test(f)); + assert.ok( + pinFiles.some((f) => f.endsWith('doc-consistency.test.mjs')), + 'doc-consistency.test.mjs must exist as the canonical prose-pin bucket', + ); +});