fix(end-state): the D-03/D-04 probes RUN a named test instead of grepping for a symbol
Measured 2026-09-18 (PM checkpoint on6cafb4c, mutant M6): a two-line file export function runPlanVerification() { return { ok: true }; } export function formatCriteriaEvidence() { return ''; } dropped into an otherwise UNFIXED tree closed both D-03 and D-04, and the gate reported "defects 0 of 7, registry integrity: intact". The same checkpoint felled 5 of 5 mutants with the SUITE and 0 of 5 with the GATE. Both probes called themselves `behaviour` while asking a grep whether an export name was present - a phrase probe pointed at code. A check condition may now name a test instead of a pattern: { "test": "tests/lib/criteria-runner.test.mjs", "name": "<exact test name>", "expect": "fails" } The gate spawns the test runner on that one test and reads its TAP line. `expect: "fails"` holds - the defect stays open - while the test does not pass. A missing file, a name that matches nothing, a runner that will not start: all throw, which counts as NOT FELLABLE and therefore open. A check that cannot fire is never "fixed". The child's `NODE_TEST_*` env is stripped. The gate usually runs UNDER the test runner, and an inherited `NODE_TEST_CONTEXT` makes the grandchild report over the parent's IPC channel instead of stdout - the TAP line would never arrive and the probe would silently stop felling anything. M6 is now a permanent mutant in the gate's own test, so the cosmetic close cannot come back. Verified end to end against a real stub tree: git archive8d1669e| tar -x -C $TMP cp scripts/end-state-gate.mjs scripts/end-state-registry.json $TMP/scripts/ cp tests/fixtures/end-state-frozen.json $TMP/tests/fixtures/; cp STATE.md $TMP/ printf 'export function runPlanVerification…' > $TMP/lib/cosmetic/stub.mjs node scripts/end-state-gate.mjs --root $TMP -> defects 2 of 7, both NOT FELLABLE, exit 1 The gate also now states the limit out loud, as its own line above the table: a behaviour probe proves the capability WORKS; that Phase 7 and Phase 4.5 CALL it is pinned by TEXT in doc-consistency, not proven deterministically - real proof is a headless plugin-eval run against a fixture plan (week 40). FROZEN DENOMINATOR AMENDED, second time in one day, by the work order that carries this change: D-03 and D-04 signatures moved because their checks changed on purpose. tests/fixtures/end-state-frozen.json and the FROZEN literal in tests/scripts/end-state-gate.test.mjs are updated together, both dated, both saying why. No entry was removed. Gate after this change: defects 0 of 7, registry integrity intact, exit 1. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
106dcb0091
commit
a7af76e5ff
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
4 changed files with 158 additions and 15 deletions
|
|
@ -75,7 +75,64 @@ function sectionOf(text, heading, label) {
|
|||
return lines.slice(start, end).join('\n');
|
||||
}
|
||||
|
||||
export const TEST_TIMEOUT_MS = 120_000;
|
||||
|
||||
const escapeRegExp = (t) => String(t).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
|
||||
/**
|
||||
* A condition that RUNS a named test and reads its result.
|
||||
*
|
||||
* A grep condition can only ask whether a string is present, so a "behaviour"
|
||||
* probe built from one is a phrase probe pointed at code: measured 2026-09-18,
|
||||
* a two-line file exporting `runPlanVerification` and `formatCriteriaEvidence`
|
||||
* closed D-03 and D-04 on a tree where nothing was fixed. This spawns the test
|
||||
* runner on one named test and reads the TAP line for it.
|
||||
*
|
||||
* `expect: "passes"` holds when the named test passes; `"fails"` holds when it
|
||||
* does not. A test file that is missing, a name that matches nothing, a runner
|
||||
* that cannot start: all THROW, which the caller counts as NOT FELLABLE and
|
||||
* therefore open. A check that cannot fire is never "fixed".
|
||||
*/
|
||||
export function evaluateTestCondition(root, cond) {
|
||||
if (cond.expect !== 'passes' && cond.expect !== 'fails') {
|
||||
throw new Error(`a test condition's expect must be "passes" or "fails", got ${JSON.stringify(cond.expect)}`);
|
||||
}
|
||||
if (typeof cond.name !== 'string' || cond.name.trim() === '') {
|
||||
throw new Error('a test condition must name exactly one test');
|
||||
}
|
||||
const file = join(root, cond.test);
|
||||
if (!existsSync(file)) throw new Error(`test file not found: ${cond.test}`);
|
||||
// The gate itself often runs UNDER the test runner (the gate's own test
|
||||
// spawns it). `NODE_TEST_CONTEXT` inherited into this child makes it report
|
||||
// over the parent's IPC channel instead of stdout, and the TAP line would
|
||||
// never arrive — a probe that silently stops fellling anything.
|
||||
const env = { ...process.env };
|
||||
for (const key of Object.keys(env)) {
|
||||
if (key.startsWith('NODE_TEST_')) delete env[key];
|
||||
}
|
||||
const r = spawnSync(
|
||||
process.execPath,
|
||||
['--test', '--test-reporter=tap', `--test-name-pattern=^${escapeRegExp(cond.name)}$`, file],
|
||||
{ cwd: root, encoding: 'utf8', timeout: TEST_TIMEOUT_MS, env },
|
||||
);
|
||||
if (r.error) throw new Error(`test runner did not run ${cond.test}: ${r.error.message}`);
|
||||
// The file itself reports as a subtest too, so the summary counters are
|
||||
// ambiguous; the TAP line carrying the exact name is not. TAP escapes `#`
|
||||
// and `\`, so unescape before comparing.
|
||||
let passed = null;
|
||||
for (const line of String(r.stdout ?? '').split('\n')) {
|
||||
const m = line.match(/^\s*(not )?ok \d+ - (.*)$/);
|
||||
if (!m) continue;
|
||||
if (m[2].replace(/\\(.)/g, '$1').trim() !== cond.name) continue;
|
||||
passed = m[1] === undefined;
|
||||
break;
|
||||
}
|
||||
if (passed === null) throw new Error(`no test named "${cond.name}" ran in ${cond.test}`);
|
||||
return cond.expect === 'passes' ? passed : !passed;
|
||||
}
|
||||
|
||||
export function evaluateCondition(root, cond) {
|
||||
if (cond.test !== undefined) return evaluateTestCondition(root, cond);
|
||||
if (cond.expect !== 'match' && cond.expect !== 'no-match') {
|
||||
throw new Error(`expect must be "match" or "no-match", got ${JSON.stringify(cond.expect)}`);
|
||||
}
|
||||
|
|
@ -164,11 +221,20 @@ const notMeasured = (detail) => ({ open: null, total: null, items: [], detail })
|
|||
// gate states the limit out loud, because a closed check is only ever as strong
|
||||
// as the thing it reads. Every kind a registry entry may declare belongs here —
|
||||
// the entry-shape test rejects a kind the gate cannot explain.
|
||||
// What no probe in this file can prove. /trekexecute and /trekreview are
|
||||
// markdown a model interprets, so no unit test can observe that Phase 7 or
|
||||
// Phase 4.5 was performed. The gate says this out loud rather than letting a
|
||||
// green defect row imply it.
|
||||
export const WIRING_NOTE =
|
||||
'wiring: a behaviour probe proves the capability WORKS. That /trekexecute Phase 7 and ' +
|
||||
'/trekreview Phase 4.5 actually CALL it is pinned by TEXT in tests/lib/doc-consistency.test.mjs, ' +
|
||||
'not proven deterministically - real proof is a headless plugin-eval run against a fixture plan (week 40).';
|
||||
|
||||
export const PROBE_NOTES = {
|
||||
phrase:
|
||||
'a phrase probe closes on rewording: a closed phrase probe is evidence, not proof of behaviour',
|
||||
behaviour:
|
||||
'a behaviour probe reads code a test exercises: it proves the capability exists, not that a prose phase calls it (the wiring is pinned by the suite, not by this probe)',
|
||||
'a behaviour probe RUNS a named test and reads its result, so a stub that only exports the right symbol does not close it; it proves the capability WORKS, not that a prose phase calls it',
|
||||
byte:
|
||||
"a byte probe reads the file's bytes, so it closes only on a real change to them",
|
||||
};
|
||||
|
|
@ -343,7 +409,7 @@ export function measure(root, registry, frozen) {
|
|||
} catch (err) {
|
||||
integrity = { ok: null, violations: [], detail: `not measurable: cannot read ${FROZEN_PATH}: ${err.message}` };
|
||||
}
|
||||
return { green: rows.every((r) => r.open === 0) && integrity.ok === true, rows, integrity };
|
||||
return { green: rows.every((r) => r.open === 0) && integrity.ok === true, rows, integrity, wiring: WIRING_NOTE };
|
||||
}
|
||||
|
||||
export function loadRegistry(root) {
|
||||
|
|
@ -376,6 +442,7 @@ export function render(result) {
|
|||
} else if (integrity) {
|
||||
out.push(`registry integrity: n/a — ${integrity.detail}`);
|
||||
}
|
||||
out.push(result.wiring ?? WIRING_NOTE);
|
||||
out.push('');
|
||||
out.push('| tally | open | of | target | source |');
|
||||
out.push('|---|---|---|---|---|');
|
||||
|
|
|
|||
|
|
@ -35,26 +35,26 @@
|
|||
{
|
||||
"id": "D-03",
|
||||
"summary": "commands/trekexecute.md never runs a trekplan's `## Verification` (where the brief's success criteria land) on the single-session path: Phase 7 says 'Skip for trekplans', and only the multi-session wave path runs master verification",
|
||||
"closesWhen": "a runnable plan-verification runner exists in lib/ (behaviour probe: it proves the capability exists and is exercised by a test, not that Phase 7 calls it — the wiring is pinned by tests/lib/doc-consistency.test.mjs)",
|
||||
"closesWhen": "a named test PROVES the single-session path fells a run on a criterion that does not hold (behaviour probe: the gate RUNS tests/lib/criteria-runner.test.mjs, so a stub that only exports the symbol does not close it; that Phase 7 CALLS it is pinned by tests/lib/doc-consistency.test.mjs, not by this probe)",
|
||||
"probe": "behaviour",
|
||||
"check": [
|
||||
{
|
||||
"path": "lib/**/*.mjs",
|
||||
"pattern": "export function runPlanVerification",
|
||||
"expect": "no-match"
|
||||
"test": "tests/lib/criteria-runner.test.mjs",
|
||||
"name": "runPlanVerification: a plan whose success criterion FAILS fells the run",
|
||||
"expect": "fails"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "D-04",
|
||||
"summary": "agents/brief-conformance-reviewer.md must judge whether a success criterion's verification command 'exists and passes', but its tools are Read/Glob/Grep, so it cannot run anything",
|
||||
"closesWhen": "the evidence block the conformance reviewer judges is BUILT IN CODE (behaviour probe: it proves /trekreview can hand over a real exit code instead of asking a Read/Glob/Grep agent whether a command passes; the rubric rewrite and the hand-over are pinned by tests/lib/doc-consistency.test.mjs)",
|
||||
"closesWhen": "a named test PROVES the evidence block the conformance reviewer judges is BUILT IN CODE from real exit codes (behaviour probe: the gate RUNS tests/lib/criteria-runner.test.mjs, so a stub that only exports the symbol does not close it; that Phase 4.5 CALLS it is pinned by tests/lib/doc-consistency.test.mjs, not by this probe)",
|
||||
"probe": "behaviour",
|
||||
"check": [
|
||||
{
|
||||
"path": "lib/**/*.mjs",
|
||||
"pattern": "export function formatCriteriaEvidence",
|
||||
"expect": "no-match"
|
||||
"test": "tests/lib/criteria-runner.test.mjs",
|
||||
"name": "formatCriteriaEvidence: one row per criterion, with command and exit code",
|
||||
"expect": "fails"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
|
|||
6
tests/fixtures/end-state-frozen.json
vendored
6
tests/fixtures/end-state-frozen.json
vendored
|
|
@ -1,10 +1,10 @@
|
|||
{
|
||||
"why": "Frozen 2026-09-17, when the end-state direction was chosen. These ids and check signatures are the end-state gate's denominator: an entry may CLOSE, it may never disappear or have its check changed. Changing this file is a decision, not a refactor - tests/scripts/end-state-gate.test.mjs pins it literally. Amended 2026-09-18 by operator decision: D-03 and D-04 were rewritten from phrase probes to behaviour probes, so both signatures moved on purpose. That is the mechanism working, not a bypass - the list may change, but only as a decision that is said out loud.",
|
||||
"why": "Frozen 2026-09-17, when the end-state direction was chosen. These ids and check signatures are the end-state gate's denominator: an entry may CLOSE, it may never disappear or have its check changed. Changing this file is a decision, not a refactor - tests/scripts/end-state-gate.test.mjs pins it literally. Amended 2026-09-18 by operator decision: D-03 and D-04 were rewritten from phrase probes to behaviour probes, so both signatures moved on purpose. That is the mechanism working, not a bypass - the list may change, but only as a decision that is said out loud. Amended again 2026-09-18 (same day, PM checkpoint follow-up, operator-authorised by work order): the D-03/D-04 behaviour probes were greps for an export name, and a two-line stub exporting both names closed them on a tree where nothing was fixed. They now RUN a named test. Both signatures moved on purpose, and the M6 stub mutant is kept as a permanent test so the cosmetic close cannot return.",
|
||||
"defects": {
|
||||
"D-01": "48f7f5cca070091e8f9b9ca0c7cdb7a1880733691935c25d07ea1d3e8d508fc9",
|
||||
"D-02": "dfc94dba04a9116ae9be2097a0c5a9d8313b02c7de1ca6b49a472c1823a6d1e1",
|
||||
"D-03": "20d2837af1a1247220c65e2a04069b41a3e922c2452369294da4d62bb985636b",
|
||||
"D-04": "200a142e7792eb00a0f0a58bbfb3e548585d9b45ba36a5c5bf94f28f2c896ffb",
|
||||
"D-03": "4eb4c7e447d927b389c7537c2e76d10e6c53ae46c12a61aed1c19adde2ffeb08",
|
||||
"D-04": "32e3c2aba4b1dc8c41581bf849f39658552e4e8b251a40824bf9dd75c846d19a",
|
||||
"D-05": "8fe6df52b43496c3302400507cb98002dcf8d2a12d92085e14bda4c7819bc7ba",
|
||||
"D-06": "72a787c8154d1c18849e9856dcf1a67703d6b5541f88db81d6bab13cab14b418",
|
||||
"D-07": "1bf3e0cd36c621720697475b0a3fd7db78611d66b58a9ce035e7695e57c69f1b"
|
||||
|
|
|
|||
|
|
@ -43,8 +43,12 @@ const FROZEN = {
|
|||
'D-01': '48f7f5cca070091e8f9b9ca0c7cdb7a1880733691935c25d07ea1d3e8d508fc9',
|
||||
'D-02': 'dfc94dba04a9116ae9be2097a0c5a9d8313b02c7de1ca6b49a472c1823a6d1e1',
|
||||
// Amended 2026-09-18 (operator decision): D-03/D-04 became behaviour probes.
|
||||
'D-03': '20d2837af1a1247220c65e2a04069b41a3e922c2452369294da4d62bb985636b',
|
||||
'D-04': '200a142e7792eb00a0f0a58bbfb3e548585d9b45ba36a5c5bf94f28f2c896ffb',
|
||||
// Amended again the same day, by work order after the PM checkpoint: those
|
||||
// behaviour probes were greps for an export name, and a two-line stub
|
||||
// exporting both names closed them on an unfixed tree (mutant M6, pinned
|
||||
// below). They now RUN a named test, so both signatures moved on purpose.
|
||||
'D-03': '4eb4c7e447d927b389c7537c2e76d10e6c53ae46c12a61aed1c19adde2ffeb08',
|
||||
'D-04': '32e3c2aba4b1dc8c41581bf849f39658552e4e8b251a40824bf9dd75c846d19a',
|
||||
'D-05': '8fe6df52b43496c3302400507cb98002dcf8d2a12d92085e14bda4c7819bc7ba',
|
||||
'D-06': '72a787c8154d1c18849e9856dcf1a67703d6b5541f88db81d6bab13cab14b418',
|
||||
'D-07': '1bf3e0cd36c621720697475b0a3fd7db78611d66b58a9ce035e7695e57c69f1b',
|
||||
|
|
@ -759,3 +763,75 @@ test('the detail line carries one note per probe kind PRESENT, and none for kind
|
|||
assert.equal(rowById(out, 'experiments').detail, '', 'an empty row explains nothing');
|
||||
} finally { cleanup(dir); }
|
||||
});
|
||||
|
||||
// --- a probe that RUNS a test, and the M6 mutant ---------------------------
|
||||
//
|
||||
// Measured 2026-09-18 (PM checkpoint on 6cafb4c): a two-line file
|
||||
// export function runPlanVerification() { return { ok: true }; }
|
||||
// export function formatCriteriaEvidence() { return ''; }
|
||||
// dropped into an otherwise UNFIXED tree closed both D-03 and D-04, and the
|
||||
// gate reported "defects 0 of 7, registry intact". A grep condition can only
|
||||
// ask whether a string is present, so a behaviour probe built from one is a
|
||||
// phrase probe pointed at code. A condition that RUNS a named test and reads
|
||||
// its TAP result cannot be closed that way.
|
||||
|
||||
const M6_STUB = [
|
||||
'export function runPlanVerification() { return { ok: true }; }',
|
||||
"export function formatCriteriaEvidence() { return ''; }",
|
||||
'',
|
||||
].join('\n');
|
||||
|
||||
const TEST_FILE = [
|
||||
"import { test } from 'node:test';",
|
||||
"import { strict as assert } from 'node:assert';",
|
||||
"test('green one', () => { assert.ok(true); });",
|
||||
"test('red one', () => { assert.ok(false, 'red on purpose'); });",
|
||||
'',
|
||||
].join('\n');
|
||||
|
||||
test('a test condition RUNS the named test and reads its result', () => {
|
||||
const dir = fixture({ 'tests/t.test.mjs': TEST_FILE });
|
||||
try {
|
||||
const cond = (name, expect) => ({ test: 'tests/t.test.mjs', name, expect });
|
||||
assert.equal(evaluateCondition(dir, cond('green one', 'passes')), true);
|
||||
assert.equal(evaluateCondition(dir, cond('green one', 'fails')), false);
|
||||
assert.equal(evaluateCondition(dir, cond('red one', 'passes')), false);
|
||||
assert.equal(evaluateCondition(dir, cond('red one', 'fails')), true);
|
||||
} finally { cleanup(dir); }
|
||||
});
|
||||
|
||||
test('a test condition naming a test that does not run is NOT FELLABLE, never closed', () => {
|
||||
const dir = fixture({ 'tests/t.test.mjs': TEST_FILE });
|
||||
try {
|
||||
for (const cond of [
|
||||
{ test: 'tests/t.test.mjs', name: 'no such test', expect: 'fails' },
|
||||
{ test: 'tests/missing.test.mjs', name: 'green one', expect: 'fails' },
|
||||
]) {
|
||||
const r = evaluateCheck(dir, [cond]);
|
||||
assert.equal(r.status, 'not-fellable', JSON.stringify(cond));
|
||||
assert.ok(r.detail.length > 0, 'the gate must say WHY the check could not fire');
|
||||
}
|
||||
} finally { cleanup(dir); }
|
||||
});
|
||||
|
||||
test('M6 mutant: a stub exporting both symbols does NOT close D-03 or D-04', () => {
|
||||
const dir = fixture({ 'lib/cosmetic/stub.mjs': M6_STUB });
|
||||
try {
|
||||
const entries = loadRegistry(ROOT).defects.filter((e) => e.id === 'D-03' || e.id === 'D-04');
|
||||
assert.equal(entries.length, 2, 'D-03 and D-04 are still in the registry');
|
||||
for (const e of entries) {
|
||||
assert.notEqual(
|
||||
evaluateCheck(dir, e.check).status, 'closed',
|
||||
`${e.id} closed on a tree where nothing is fixed: ${JSON.stringify(e.check)}`,
|
||||
);
|
||||
}
|
||||
} finally { cleanup(dir); }
|
||||
});
|
||||
|
||||
test('the gate states out loud that the wiring is pinned by text, not proven', () => {
|
||||
const r = spawnSync(process.execPath, [GATE], { encoding: 'utf8', cwd: ROOT });
|
||||
assert.ok(r.status === 0 || r.status === 1, r.stderr);
|
||||
assert.match(r.stdout, /wiring: /);
|
||||
assert.match(r.stdout, /pinned by TEXT/);
|
||||
assert.match(r.stdout, /plugin-eval/);
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue