fix(end-state): a behaviour probe must FELL a stub, not just name a test
The cosmetic close came back one level up. The probe reads the test file from the same tree it measures, so a four-line file holding two EMPTY tests with the two named names closed D-03 and D-04 on a tree where `lib/verification/` did not exist at all - "defects 0 of 7, registry intact" (measured 2026-09-18). First a two-line stub exporting the right symbols; then an empty test with the right name. A name is always forgeable. So the named test is now run twice. Once on the tree, as before - and once in a sandbox where the module the condition declares in `stubs` is replaced by a stub exporting the same names, all inert. If the test still passes there, it binds the name and not the behaviour, and the condition THROWS: NOT FELLABLE, counted open. A condition that declares no `stubs`, or names a module that is not there, cannot fire either. The sandbox is a symlink overlay: every entry of the tree is symlinked, and only the test file and the stubbed module are materialised for real - Node resolves an ESM import through the realpath, so a symlinked test file would import the original module and never see the mutant. Nothing is ever written inside the measured tree, and the only directory removed is the one this code made under the system temp dir (pinned by a test). M7 is now a permanent mutant beside M6, in two forms: the checkpoint's own reproduction (unfixed tree + empty named tests) and the harder one (the real module present, so the stub can be built and the empty test passes against it). Both report `defects 2 of 7`. A positive control pins that D-03/D-04 still CLOSE on the real tree, so "not closed" everywhere cannot read as a working probe. The frozen denominator moves a third time, deliberately, and its `why` no longer claims authority it does not have: the second and third amendments were maintenance decisions by the maintainer, not operator decisions, and the tracked file now says exactly that. Measured after: real tree node scripts/end-state-gate.mjs -> defects 0 of 7, intact, exit 1 M6 (stub)8d1669e+ current gate/registry/frozen + lib/cosmetic/stub.mjs -> 2 of 7 M7 (empty)8d1669e+ current gate/registry/frozen + 3-line tests/lib/criteria-runner.test.mjs with the two named tests -> 2 of 7 Red first: 4 of the new tests failed before the change. Suite 1154 (1152/0/2). Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
02243c6365
commit
83d82f121d
4 changed files with 252 additions and 45 deletions
|
|
@ -19,7 +19,11 @@
|
||||||
//
|
//
|
||||||
// Usage: node scripts/end-state-gate.mjs [--json] [--root <dir>]
|
// Usage: node scripts/end-state-gate.mjs [--json] [--root <dir>]
|
||||||
|
|
||||||
import { readFileSync, existsSync, readdirSync, statSync, realpathSync } from 'node:fs';
|
import {
|
||||||
|
readFileSync, writeFileSync, existsSync, readdirSync, statSync, realpathSync,
|
||||||
|
mkdirSync, mkdtempSync, symlinkSync, rmSync,
|
||||||
|
} from 'node:fs';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
import { join, resolve, dirname, relative, sep, basename } from 'node:path';
|
import { join, resolve, dirname, relative, sep, basename } from 'node:path';
|
||||||
import { fileURLToPath } from 'node:url';
|
import { fileURLToPath } from 'node:url';
|
||||||
import { createHash } from 'node:crypto';
|
import { createHash } from 'node:crypto';
|
||||||
|
|
@ -79,19 +83,103 @@ export const TEST_TIMEOUT_MS = 120_000;
|
||||||
|
|
||||||
const escapeRegExp = (t) => String(t).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
const escapeRegExp = (t) => String(t).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||||
|
|
||||||
|
// Every `export` name a module declares, so a stub can offer the same surface
|
||||||
|
// and none of the behaviour.
|
||||||
|
const EXPORT_NAME = /^\s*export\s+(?:async\s+)?(?:function\*?|const|let|var|class)\s+([A-Za-z_$][\w$]*)/gm;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A condition that RUNS a named test and reads its result.
|
* A stub of a module: the same export names, every one of them inert.
|
||||||
|
*
|
||||||
|
* This is the mutant a behaviour probe must FELL. A test that still passes
|
||||||
|
* when the module it names does nothing is not measuring that module — the
|
||||||
|
* empty test that closed D-03 and D-04 on a tree without `lib/verification/`
|
||||||
|
* is the limiting case (measured 2026-09-18).
|
||||||
|
*/
|
||||||
|
export function stubModule(source) {
|
||||||
|
const names = new Set();
|
||||||
|
for (const m of String(source).matchAll(EXPORT_NAME)) names.add(m[1]);
|
||||||
|
const lines = [...names].map((n) => `export const ${n} = undefined;`);
|
||||||
|
if (/^\s*export\s+default\b/m.test(source)) lines.push('export default undefined;');
|
||||||
|
return `${lines.join('\n')}\n`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build `dest` as a shallow copy of `root`: every entry is a SYMLINK to the
|
||||||
|
// original, except the files in `files`, which are written for real (and whose
|
||||||
|
// parent directories are therefore real directories too). Node resolves an
|
||||||
|
// ESM import through the realpath, so a symlinked test file would import the
|
||||||
|
// ORIGINAL module and never see the stub — only the files that must differ are
|
||||||
|
// materialised. Nothing is ever written inside the measured tree.
|
||||||
|
function overlay(root, dest, files) {
|
||||||
|
const realDirs = new Set();
|
||||||
|
for (const rel of files.keys()) {
|
||||||
|
const parts = rel.split('/');
|
||||||
|
for (let i = 0; i < parts.length - 1; i++) realDirs.add(parts.slice(0, i + 1).join('/'));
|
||||||
|
}
|
||||||
|
const build = (relDir) => {
|
||||||
|
const absSrc = relDir === '' ? root : join(root, relDir);
|
||||||
|
const absDst = relDir === '' ? dest : join(dest, relDir);
|
||||||
|
mkdirSync(absDst, { recursive: true });
|
||||||
|
if (!existsSync(absSrc)) return;
|
||||||
|
for (const name of readdirSync(absSrc)) {
|
||||||
|
const childRel = relDir === '' ? name : `${relDir}/${name}`;
|
||||||
|
if (realDirs.has(childRel)) { build(childRel); continue; }
|
||||||
|
if (files.has(childRel)) continue;
|
||||||
|
symlinkSync(join(absSrc, name), join(absDst, name));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
build('');
|
||||||
|
for (const [rel, body] of files) writeFileSync(join(dest, rel), body);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run ONE named test out of one test file and report whether it passed.
|
||||||
|
// `null` = the runner produced no TAP line for that name (it did not run at
|
||||||
|
// all), which is never a pass.
|
||||||
|
function runNamedTest(root, testRel, name) {
|
||||||
|
// The gate itself often runs UNDER the test runner (the gate's own test
|
||||||
|
// spawns it). `NODE_TEST_CONTEXT` inherited into this child makes it report
|
||||||
|
// over the parent's IPC channel instead of stdout, and the TAP line would
|
||||||
|
// never arrive — a probe that silently stops felling anything.
|
||||||
|
const env = { ...process.env };
|
||||||
|
for (const key of Object.keys(env)) {
|
||||||
|
if (key.startsWith('NODE_TEST_')) delete env[key];
|
||||||
|
}
|
||||||
|
const r = spawnSync(
|
||||||
|
process.execPath,
|
||||||
|
['--test', '--test-reporter=tap', `--test-name-pattern=^${escapeRegExp(name)}$`, testRel],
|
||||||
|
{ cwd: root, encoding: 'utf8', timeout: TEST_TIMEOUT_MS, env },
|
||||||
|
);
|
||||||
|
if (r.error) throw new Error(`test runner did not run ${testRel}: ${r.error.message}`);
|
||||||
|
// The file itself reports as a subtest too, so the summary counters are
|
||||||
|
// ambiguous; the TAP line carrying the exact name is not. TAP escapes `#`
|
||||||
|
// and `\`, so unescape before comparing.
|
||||||
|
for (const line of String(r.stdout ?? '').split('\n')) {
|
||||||
|
const m = line.match(/^\s*(not )?ok \d+ - (.*)$/);
|
||||||
|
if (!m) continue;
|
||||||
|
if (m[2].replace(/\\(.)/g, '$1').trim() !== name) continue;
|
||||||
|
return m[1] === undefined;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A condition that RUNS a named test, reads its result, and requires that the
|
||||||
|
* same test FELLS a stub of the module it claims to bind.
|
||||||
*
|
*
|
||||||
* A grep condition can only ask whether a string is present, so a "behaviour"
|
* A grep condition can only ask whether a string is present, so a "behaviour"
|
||||||
* probe built from one is a phrase probe pointed at code: measured 2026-09-18,
|
* probe built from one is a phrase probe pointed at code: measured
|
||||||
* a two-line file exporting `runPlanVerification` and `formatCriteriaEvidence`
|
* 2026-09-18, a two-line file exporting `runPlanVerification` and
|
||||||
* closed D-03 and D-04 on a tree where nothing was fixed. This spawns the test
|
* `formatCriteriaEvidence` closed D-03 and D-04 on a tree where nothing was
|
||||||
* runner on one named test and reads the TAP line for it.
|
* fixed. Running a NAMED test closed that hole and opened a smaller one one
|
||||||
|
* level up: two EMPTY tests with the right names closed them again. A name is
|
||||||
|
* always forgeable; felling a mutant is not. So the named test is run twice —
|
||||||
|
* once on the tree, once against a stub of each module in `stubs` — and it
|
||||||
|
* must not pass the second time.
|
||||||
*
|
*
|
||||||
* `expect: "passes"` holds when the named test passes; `"fails"` holds when it
|
* `expect: "passes"` holds when the named test passes; `"fails"` holds when it
|
||||||
* does not. A test file that is missing, a name that matches nothing, a runner
|
* does not. A test file that is missing, a name that matches nothing, a runner
|
||||||
* that cannot start: all THROW, which the caller counts as NOT FELLABLE and
|
* that cannot start, a missing or unfelled stub target: all THROW, which the
|
||||||
* therefore open. A check that cannot fire is never "fixed".
|
* caller counts as NOT FELLABLE and therefore open. A check that cannot fire
|
||||||
|
* is never "fixed".
|
||||||
*/
|
*/
|
||||||
export function evaluateTestCondition(root, cond) {
|
export function evaluateTestCondition(root, cond) {
|
||||||
if (cond.expect !== 'passes' && cond.expect !== 'fails') {
|
if (cond.expect !== 'passes' && cond.expect !== 'fails') {
|
||||||
|
|
@ -100,33 +188,33 @@ export function evaluateTestCondition(root, cond) {
|
||||||
if (typeof cond.name !== 'string' || cond.name.trim() === '') {
|
if (typeof cond.name !== 'string' || cond.name.trim() === '') {
|
||||||
throw new Error('a test condition must name exactly one test');
|
throw new Error('a test condition must name exactly one test');
|
||||||
}
|
}
|
||||||
|
if (!Array.isArray(cond.stubs) || cond.stubs.length === 0) {
|
||||||
|
throw new Error('a test condition must declare `stubs`: the module(s) whose stub the named test must fell');
|
||||||
|
}
|
||||||
const file = join(root, cond.test);
|
const file = join(root, cond.test);
|
||||||
if (!existsSync(file)) throw new Error(`test file not found: ${cond.test}`);
|
if (!existsSync(file)) throw new Error(`test file not found: ${cond.test}`);
|
||||||
// The gate itself often runs UNDER the test runner (the gate's own test
|
|
||||||
// spawns it). `NODE_TEST_CONTEXT` inherited into this child makes it report
|
for (const target of cond.stubs) {
|
||||||
// over the parent's IPC channel instead of stdout, and the TAP line would
|
const abs = join(root, target);
|
||||||
// never arrive — a probe that silently stops fellling anything.
|
if (!existsSync(abs)) throw new Error(`stub target not found: ${target}`);
|
||||||
const env = { ...process.env };
|
const sandbox = mkdtempSync(join(tmpdir(), 'end-state-mutant-'));
|
||||||
for (const key of Object.keys(env)) {
|
try {
|
||||||
if (key.startsWith('NODE_TEST_')) delete env[key];
|
overlay(root, sandbox, new Map([
|
||||||
}
|
[cond.test, readFileSync(file, 'utf8')],
|
||||||
const r = spawnSync(
|
[target, stubModule(readFileSync(abs, 'utf8'))],
|
||||||
process.execPath,
|
]));
|
||||||
['--test', '--test-reporter=tap', `--test-name-pattern=^${escapeRegExp(cond.name)}$`, file],
|
if (runNamedTest(sandbox, cond.test, cond.name) === true) {
|
||||||
{ cwd: root, encoding: 'utf8', timeout: TEST_TIMEOUT_MS, env },
|
throw new Error(
|
||||||
|
`"${cond.name}" still passes against a stub of ${target} — it binds the name, not the behaviour`,
|
||||||
);
|
);
|
||||||
if (r.error) throw new Error(`test runner did not run ${cond.test}: ${r.error.message}`);
|
|
||||||
// The file itself reports as a subtest too, so the summary counters are
|
|
||||||
// ambiguous; the TAP line carrying the exact name is not. TAP escapes `#`
|
|
||||||
// and `\`, so unescape before comparing.
|
|
||||||
let passed = null;
|
|
||||||
for (const line of String(r.stdout ?? '').split('\n')) {
|
|
||||||
const m = line.match(/^\s*(not )?ok \d+ - (.*)$/);
|
|
||||||
if (!m) continue;
|
|
||||||
if (m[2].replace(/\\(.)/g, '$1').trim() !== cond.name) continue;
|
|
||||||
passed = m[1] === undefined;
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
|
} finally {
|
||||||
|
// Only ever a directory this function made, under the system temp dir.
|
||||||
|
if (sandbox.startsWith(realpathSync(tmpdir()))) rmSync(sandbox, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const passed = runNamedTest(root, cond.test, cond.name);
|
||||||
if (passed === null) throw new Error(`no test named "${cond.name}" ran in ${cond.test}`);
|
if (passed === null) throw new Error(`no test named "${cond.name}" ran in ${cond.test}`);
|
||||||
return cond.expect === 'passes' ? passed : !passed;
|
return cond.expect === 'passes' ? passed : !passed;
|
||||||
}
|
}
|
||||||
|
|
@ -234,7 +322,7 @@ export const PROBE_NOTES = {
|
||||||
phrase:
|
phrase:
|
||||||
'a phrase probe closes on rewording: a closed phrase probe is evidence, not proof of behaviour',
|
'a phrase probe closes on rewording: a closed phrase probe is evidence, not proof of behaviour',
|
||||||
behaviour:
|
behaviour:
|
||||||
'a behaviour probe RUNS a named test and reads its result, so a stub that only exports the right symbol does not close it; it proves the capability WORKS, not that a prose phase calls it',
|
'a behaviour probe RUNS a named test AND re-runs it against a stub of the module it binds, which it must fell: neither a stub that exports the right symbols nor an empty test with the right name closes it; it proves the capability WORKS, not that a prose phase calls it',
|
||||||
byte:
|
byte:
|
||||||
"a byte probe reads the file's bytes, so it closes only on a real change to them",
|
"a byte probe reads the file's bytes, so it closes only on a real change to them",
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -35,26 +35,32 @@
|
||||||
{
|
{
|
||||||
"id": "D-03",
|
"id": "D-03",
|
||||||
"summary": "commands/trekexecute.md never runs a trekplan's `## Verification` (where the brief's success criteria land) on the single-session path: Phase 7 says 'Skip for trekplans', and only the multi-session wave path runs master verification",
|
"summary": "commands/trekexecute.md never runs a trekplan's `## Verification` (where the brief's success criteria land) on the single-session path: Phase 7 says 'Skip for trekplans', and only the multi-session wave path runs master verification",
|
||||||
"closesWhen": "a named test PROVES the single-session path fells a run on a criterion that does not hold (behaviour probe: the gate RUNS tests/lib/criteria-runner.test.mjs, so a stub that only exports the symbol does not close it; that Phase 7 CALLS it is pinned by tests/lib/doc-consistency.test.mjs, not by this probe)",
|
"closesWhen": "a named test PROVES the single-session path fells a run on a criterion that does not hold (behaviour probe: the gate RUNS tests/lib/criteria-runner.test.mjs, so a stub that only exports the symbol does not close it, and the named test must itself FELL a stub of lib/verification/criteria-runner.mjs, so an EMPTY test with the right name does not close it either; that Phase 7 CALLS it is pinned by tests/lib/doc-consistency.test.mjs, not by this probe)",
|
||||||
"probe": "behaviour",
|
"probe": "behaviour",
|
||||||
"check": [
|
"check": [
|
||||||
{
|
{
|
||||||
"test": "tests/lib/criteria-runner.test.mjs",
|
"test": "tests/lib/criteria-runner.test.mjs",
|
||||||
"name": "runPlanVerification: a plan whose success criterion FAILS fells the run",
|
"name": "runPlanVerification: a plan whose success criterion FAILS fells the run",
|
||||||
"expect": "fails"
|
"expect": "fails",
|
||||||
|
"stubs": [
|
||||||
|
"lib/verification/criteria-runner.mjs"
|
||||||
|
]
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "D-04",
|
"id": "D-04",
|
||||||
"summary": "agents/brief-conformance-reviewer.md must judge whether a success criterion's verification command 'exists and passes', but its tools are Read/Glob/Grep, so it cannot run anything",
|
"summary": "agents/brief-conformance-reviewer.md must judge whether a success criterion's verification command 'exists and passes', but its tools are Read/Glob/Grep, so it cannot run anything",
|
||||||
"closesWhen": "a named test PROVES the evidence block the conformance reviewer judges is BUILT IN CODE from real exit codes (behaviour probe: the gate RUNS tests/lib/criteria-runner.test.mjs, so a stub that only exports the symbol does not close it; that Phase 4.5 CALLS it is pinned by tests/lib/doc-consistency.test.mjs, not by this probe)",
|
"closesWhen": "a named test PROVES the evidence block the conformance reviewer judges is BUILT IN CODE from real exit codes (behaviour probe: the gate RUNS tests/lib/criteria-runner.test.mjs, so a stub that only exports the symbol does not close it, and the named test must itself FELL a stub of lib/verification/criteria-runner.mjs, so an EMPTY test with the right name does not close it either; that Phase 4.5 CALLS it is pinned by tests/lib/doc-consistency.test.mjs, not by this probe)",
|
||||||
"probe": "behaviour",
|
"probe": "behaviour",
|
||||||
"check": [
|
"check": [
|
||||||
{
|
{
|
||||||
"test": "tests/lib/criteria-runner.test.mjs",
|
"test": "tests/lib/criteria-runner.test.mjs",
|
||||||
"name": "formatCriteriaEvidence: one row per criterion, with command and exit code",
|
"name": "formatCriteriaEvidence: one row per criterion, with command and exit code",
|
||||||
"expect": "fails"
|
"expect": "fails",
|
||||||
|
"stubs": [
|
||||||
|
"lib/verification/criteria-runner.mjs"
|
||||||
|
]
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|
|
||||||
6
tests/fixtures/end-state-frozen.json
vendored
6
tests/fixtures/end-state-frozen.json
vendored
|
|
@ -1,10 +1,10 @@
|
||||||
{
|
{
|
||||||
"why": "Frozen 2026-09-17, when the end-state direction was chosen. These ids and check signatures are the end-state gate's denominator: an entry may CLOSE, it may never disappear or have its check changed. Changing this file is a decision, not a refactor - tests/scripts/end-state-gate.test.mjs pins it literally. Amended 2026-09-18 by operator decision: D-03 and D-04 were rewritten from phrase probes to behaviour probes, so both signatures moved on purpose. That is the mechanism working, not a bypass - the list may change, but only as a decision that is said out loud. Amended again 2026-09-18 (same day, PM checkpoint follow-up, operator-authorised by work order): the D-03/D-04 behaviour probes were greps for an export name, and a two-line stub exporting both names closed them on a tree where nothing was fixed. They now RUN a named test. Both signatures moved on purpose, and the M6 stub mutant is kept as a permanent test so the cosmetic close cannot return.",
|
"why": "Frozen 2026-09-17, when the end-state direction was chosen. These ids and check signatures are the end-state gate's denominator: an entry may CLOSE, it may never disappear or have its check changed. Changing this file is a decision, not a refactor - tests/scripts/end-state-gate.test.mjs pins it literally. Amended 2026-09-18 by operator decision: D-03 and D-04 were rewritten from phrase probes to behaviour probes, so both signatures moved on purpose. That is the mechanism working, not a bypass - the list may change, but only as a decision that is said out loud. Amended a second time the same day, as a maintenance decision by the maintainer (no operator decision was asked for or given): the D-03/D-04 behaviour probes were greps for an export name, and a two-line stub exporting both names closed them on a tree where nothing was fixed. They now RUN a named test. Amended a third time the same day, same standing: running a NAMED test moved the cosmetic close one level up - two EMPTY tests with the right names closed both defects again - so each condition now also names the module whose STUB its test must fell. All three amendments moved signatures on purpose, removed nothing, and are kept honest by permanent mutants in the gate's own test (M6: a stub exporting the symbols; M7: an empty test with the right name).",
|
||||||
"defects": {
|
"defects": {
|
||||||
"D-01": "48f7f5cca070091e8f9b9ca0c7cdb7a1880733691935c25d07ea1d3e8d508fc9",
|
"D-01": "48f7f5cca070091e8f9b9ca0c7cdb7a1880733691935c25d07ea1d3e8d508fc9",
|
||||||
"D-02": "dfc94dba04a9116ae9be2097a0c5a9d8313b02c7de1ca6b49a472c1823a6d1e1",
|
"D-02": "dfc94dba04a9116ae9be2097a0c5a9d8313b02c7de1ca6b49a472c1823a6d1e1",
|
||||||
"D-03": "4eb4c7e447d927b389c7537c2e76d10e6c53ae46c12a61aed1c19adde2ffeb08",
|
"D-03": "98698ce74c243353713786b8604160f6672568b090884e9428d9e5e9fbcad89d",
|
||||||
"D-04": "32e3c2aba4b1dc8c41581bf849f39658552e4e8b251a40824bf9dd75c846d19a",
|
"D-04": "df11a2e95db7b014ce077db0473e5dde4195ace92d8077cacdc1cee273947b47",
|
||||||
"D-05": "8fe6df52b43496c3302400507cb98002dcf8d2a12d92085e14bda4c7819bc7ba",
|
"D-05": "8fe6df52b43496c3302400507cb98002dcf8d2a12d92085e14bda4c7819bc7ba",
|
||||||
"D-06": "72a787c8154d1c18849e9856dcf1a67703d6b5541f88db81d6bab13cab14b418",
|
"D-06": "72a787c8154d1c18849e9856dcf1a67703d6b5541f88db81d6bab13cab14b418",
|
||||||
"D-07": "1bf3e0cd36c621720697475b0a3fd7db78611d66b58a9ce035e7695e57c69f1b"
|
"D-07": "1bf3e0cd36c621720697475b0a3fd7db78611d66b58a9ce035e7695e57c69f1b"
|
||||||
|
|
|
||||||
|
|
@ -43,12 +43,17 @@ const FROZEN = {
|
||||||
'D-01': '48f7f5cca070091e8f9b9ca0c7cdb7a1880733691935c25d07ea1d3e8d508fc9',
|
'D-01': '48f7f5cca070091e8f9b9ca0c7cdb7a1880733691935c25d07ea1d3e8d508fc9',
|
||||||
'D-02': 'dfc94dba04a9116ae9be2097a0c5a9d8313b02c7de1ca6b49a472c1823a6d1e1',
|
'D-02': 'dfc94dba04a9116ae9be2097a0c5a9d8313b02c7de1ca6b49a472c1823a6d1e1',
|
||||||
// Amended 2026-09-18 (operator decision): D-03/D-04 became behaviour probes.
|
// Amended 2026-09-18 (operator decision): D-03/D-04 became behaviour probes.
|
||||||
// Amended again the same day, by work order after the PM checkpoint: those
|
// Amended a second time the same day, as a maintenance decision: those
|
||||||
// behaviour probes were greps for an export name, and a two-line stub
|
// behaviour probes were greps for an export name, and a two-line stub
|
||||||
// exporting both names closed them on an unfixed tree (mutant M6, pinned
|
// exporting both names closed them on an unfixed tree (mutant M6, pinned
|
||||||
// below). They now RUN a named test, so both signatures moved on purpose.
|
// below). They now RUN a named test.
|
||||||
'D-03': '4eb4c7e447d927b389c7537c2e76d10e6c53ae46c12a61aed1c19adde2ffeb08',
|
// Amended a third time the same day, same standing: a named test can be
|
||||||
'D-04': '32e3c2aba4b1dc8c41581bf849f39658552e4e8b251a40824bf9dd75c846d19a',
|
// EMPTY, and two empty tests with the right names closed both defects on a
|
||||||
|
// tree without lib/verification/ (mutant M7, pinned below). Each condition
|
||||||
|
// now also names the module whose STUB its test must fell. Every amendment
|
||||||
|
// moved a signature on purpose and removed nothing.
|
||||||
|
'D-03': '98698ce74c243353713786b8604160f6672568b090884e9428d9e5e9fbcad89d',
|
||||||
|
'D-04': 'df11a2e95db7b014ce077db0473e5dde4195ace92d8077cacdc1cee273947b47',
|
||||||
'D-05': '8fe6df52b43496c3302400507cb98002dcf8d2a12d92085e14bda4c7819bc7ba',
|
'D-05': '8fe6df52b43496c3302400507cb98002dcf8d2a12d92085e14bda4c7819bc7ba',
|
||||||
'D-06': '72a787c8154d1c18849e9856dcf1a67703d6b5541f88db81d6bab13cab14b418',
|
'D-06': '72a787c8154d1c18849e9856dcf1a67703d6b5541f88db81d6bab13cab14b418',
|
||||||
'D-07': '1bf3e0cd36c621720697475b0a3fd7db78611d66b58a9ce035e7695e57c69f1b',
|
'D-07': '1bf3e0cd36c621720697475b0a3fd7db78611d66b58a9ce035e7695e57c69f1b',
|
||||||
|
|
@ -781,18 +786,24 @@ const M6_STUB = [
|
||||||
'',
|
'',
|
||||||
].join('\n');
|
].join('\n');
|
||||||
|
|
||||||
|
// A fixture pair: a module, and a test that BINDS it. `green one` passes on
|
||||||
|
// the tree and fails against a stub of the module - which is what makes it a
|
||||||
|
// behaviour probe rather than a name.
|
||||||
|
const MODULE = 'export function answer() { return 42; }\n';
|
||||||
|
|
||||||
const TEST_FILE = [
|
const TEST_FILE = [
|
||||||
"import { test } from 'node:test';",
|
"import { test } from 'node:test';",
|
||||||
"import { strict as assert } from 'node:assert';",
|
"import { strict as assert } from 'node:assert';",
|
||||||
"test('green one', () => { assert.ok(true); });",
|
"import { answer } from '../lib/m.mjs';",
|
||||||
|
"test('green one', () => { assert.equal(answer(), 42); });",
|
||||||
"test('red one', () => { assert.ok(false, 'red on purpose'); });",
|
"test('red one', () => { assert.ok(false, 'red on purpose'); });",
|
||||||
'',
|
'',
|
||||||
].join('\n');
|
].join('\n');
|
||||||
|
|
||||||
test('a test condition RUNS the named test and reads its result', () => {
|
test('a test condition RUNS the named test and reads its result', () => {
|
||||||
const dir = fixture({ 'tests/t.test.mjs': TEST_FILE });
|
const dir = fixture({ 'tests/t.test.mjs': TEST_FILE, 'lib/m.mjs': MODULE });
|
||||||
try {
|
try {
|
||||||
const cond = (name, expect) => ({ test: 'tests/t.test.mjs', name, expect });
|
const cond = (name, expect) => ({ test: 'tests/t.test.mjs', name, expect, stubs: ['lib/m.mjs'] });
|
||||||
assert.equal(evaluateCondition(dir, cond('green one', 'passes')), true);
|
assert.equal(evaluateCondition(dir, cond('green one', 'passes')), true);
|
||||||
assert.equal(evaluateCondition(dir, cond('green one', 'fails')), false);
|
assert.equal(evaluateCondition(dir, cond('green one', 'fails')), false);
|
||||||
assert.equal(evaluateCondition(dir, cond('red one', 'passes')), false);
|
assert.equal(evaluateCondition(dir, cond('red one', 'passes')), false);
|
||||||
|
|
@ -801,11 +812,11 @@ test('a test condition RUNS the named test and reads its result', () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
test('a test condition naming a test that does not run is NOT FELLABLE, never closed', () => {
|
test('a test condition naming a test that does not run is NOT FELLABLE, never closed', () => {
|
||||||
const dir = fixture({ 'tests/t.test.mjs': TEST_FILE });
|
const dir = fixture({ 'tests/t.test.mjs': TEST_FILE, 'lib/m.mjs': MODULE });
|
||||||
try {
|
try {
|
||||||
for (const cond of [
|
for (const cond of [
|
||||||
{ test: 'tests/t.test.mjs', name: 'no such test', expect: 'fails' },
|
{ test: 'tests/t.test.mjs', name: 'no such test', expect: 'fails', stubs: ['lib/m.mjs'] },
|
||||||
{ test: 'tests/missing.test.mjs', name: 'green one', expect: 'fails' },
|
{ test: 'tests/missing.test.mjs', name: 'green one', expect: 'fails', stubs: ['lib/m.mjs'] },
|
||||||
]) {
|
]) {
|
||||||
const r = evaluateCheck(dir, [cond]);
|
const r = evaluateCheck(dir, [cond]);
|
||||||
assert.equal(r.status, 'not-fellable', JSON.stringify(cond));
|
assert.equal(r.status, 'not-fellable', JSON.stringify(cond));
|
||||||
|
|
@ -835,3 +846,105 @@ test('the gate states out loud that the wiring is pinned by text, not proven', (
|
||||||
assert.match(r.stdout, /pinned by TEXT/);
|
assert.match(r.stdout, /pinned by TEXT/);
|
||||||
assert.match(r.stdout, /plugin-eval/);
|
assert.match(r.stdout, /plugin-eval/);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// --- M7: a probe must bind the test's CONTENT, not its name ---------------
|
||||||
|
//
|
||||||
|
// Measured 2026-09-18 (PM checkpoint on e1e7bdf): the cosmetic close came back
|
||||||
|
// one level up. The probe reads the test file from the SAME tree it measures,
|
||||||
|
// so a four-line file holding two EMPTY tests with the two named names —
|
||||||
|
// test('runPlanVerification: a plan whose success criterion FAILS fells the run', () => {});
|
||||||
|
// test('formatCriteriaEvidence: one row per criterion, with command and exit code', () => {});
|
||||||
|
// — closed D-03 and D-04 on a tree where `lib/verification/` did not exist at
|
||||||
|
// all: "defects 0 of 7, registry intact". An empty test is the new two-line
|
||||||
|
// stub. The only form that cannot be closed by a name is one that requires the
|
||||||
|
// named test to FELL a mutant: the gate runs it a second time against a tree
|
||||||
|
// where the module it binds is replaced by a stub exporting the same names and
|
||||||
|
// doing nothing. A test that passes against both is not a behaviour probe.
|
||||||
|
|
||||||
|
const EMPTY_NAMED_TESTS = [
|
||||||
|
"import test from 'node:test';",
|
||||||
|
"test('runPlanVerification: a plan whose success criterion FAILS fells the run', () => {});",
|
||||||
|
"test('formatCriteriaEvidence: one row per criterion, with command and exit code', () => {});",
|
||||||
|
'',
|
||||||
|
].join('\n');
|
||||||
|
|
||||||
|
const REAL_RUNNER = join(ROOT, 'lib', 'verification', 'criteria-runner.mjs');
|
||||||
|
const REAL_RUNNER_TEST = join(ROOT, 'tests', 'lib', 'criteria-runner.test.mjs');
|
||||||
|
|
||||||
|
test('M7 mutant: two EMPTY tests with the right names do NOT close D-03 or D-04', () => {
|
||||||
|
// The checkpoint's reproduction: an unfixed tree (no lib/verification/) plus
|
||||||
|
// the four-line test file.
|
||||||
|
const dir = fixture({ 'tests/lib/criteria-runner.test.mjs': EMPTY_NAMED_TESTS });
|
||||||
|
try {
|
||||||
|
const entries = loadRegistry(ROOT).defects.filter((e) => e.id === 'D-03' || e.id === 'D-04');
|
||||||
|
assert.equal(entries.length, 2, 'D-03 and D-04 are still in the registry');
|
||||||
|
for (const e of entries) {
|
||||||
|
const r = evaluateCheck(dir, e.check);
|
||||||
|
assert.equal(r.status, 'not-fellable', `${e.id} closed on a tree where nothing is fixed`);
|
||||||
|
assert.ok(r.detail.length > 0, 'the gate must say WHY the check could not fire');
|
||||||
|
}
|
||||||
|
} finally { cleanup(dir); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('M7 mutant: an empty named test does not close the defect even when the module IS present', () => {
|
||||||
|
// The harder case: the real module is there, so the stub CAN be built — and
|
||||||
|
// the empty test passes against it, which is exactly the proof that the test
|
||||||
|
// binds the name and nothing else.
|
||||||
|
const dir = fixture({
|
||||||
|
'tests/lib/criteria-runner.test.mjs': EMPTY_NAMED_TESTS,
|
||||||
|
'lib/verification/criteria-runner.mjs': readFileSync(REAL_RUNNER, 'utf8'),
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
for (const e of loadRegistry(ROOT).defects.filter((x) => x.id === 'D-03' || x.id === 'D-04')) {
|
||||||
|
const r = evaluateCheck(dir, e.check);
|
||||||
|
assert.equal(r.status, 'not-fellable', `${e.id} closed on an empty test`);
|
||||||
|
assert.match(r.detail, /stub/i, 'the gate must name the mutant the test failed to fell');
|
||||||
|
}
|
||||||
|
} finally { cleanup(dir); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test("D-03/D-04 still CLOSE on the real tree — the named tests do fell a stub", () => {
|
||||||
|
// The positive control. Without it, "not closed" everywhere would read as a
|
||||||
|
// working probe when the probe is simply broken.
|
||||||
|
for (const e of loadRegistry(ROOT).defects.filter((x) => x.id === 'D-03' || x.id === 'D-04')) {
|
||||||
|
const r = evaluateCheck(ROOT, e.check);
|
||||||
|
assert.equal(r.status, 'closed', `${e.id} is not closed on the real tree: ${r.detail}`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a test condition must name the module its test binds, or it cannot fire', () => {
|
||||||
|
const dir = fixture({ 'tests/t.test.mjs': TEST_FILE, 'lib/m.mjs': MODULE });
|
||||||
|
try {
|
||||||
|
for (const cond of [
|
||||||
|
{ test: 'tests/t.test.mjs', name: 'green one', expect: 'passes' },
|
||||||
|
{ test: 'tests/t.test.mjs', name: 'green one', expect: 'passes', stubs: [] },
|
||||||
|
{ test: 'tests/t.test.mjs', name: 'green one', expect: 'passes', stubs: ['lib/gone.mjs'] },
|
||||||
|
]) {
|
||||||
|
const r = evaluateCheck(dir, [cond]);
|
||||||
|
assert.equal(r.status, 'not-fellable', JSON.stringify(cond));
|
||||||
|
assert.ok(r.detail.length > 0, 'the gate must say WHY');
|
||||||
|
}
|
||||||
|
} finally { cleanup(dir); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('the stub mutant leaves the measured tree untouched', () => {
|
||||||
|
const dir = fixture({ 'tests/t.test.mjs': TEST_FILE, 'lib/m.mjs': MODULE });
|
||||||
|
try {
|
||||||
|
const before = readFileSync(join(dir, 'lib', 'm.mjs'), 'utf8');
|
||||||
|
evaluateCheck(dir, [{ test: 'tests/t.test.mjs', name: 'green one', expect: 'passes', stubs: ['lib/m.mjs'] }]);
|
||||||
|
assert.equal(readFileSync(join(dir, 'lib', 'm.mjs'), 'utf8'), before, 'the mutation must happen in a copy, never in place');
|
||||||
|
} finally { cleanup(dir); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test('the frozen file claims no authority it does not have', () => {
|
||||||
|
// MINOR from the same checkpoint: the tracked frozen file said the second
|
||||||
|
// amendment was "operator-authorised by work order". No operator authorised
|
||||||
|
// it; it followed from a maintenance decision. A file whose whole job is to
|
||||||
|
// be an explicit decision trail may not overclaim in either direction.
|
||||||
|
const why = loadFrozen(ROOT).why;
|
||||||
|
assert.ok(why.length > 0, 'the frozen manifest must say WHY it changed');
|
||||||
|
assert.ok(
|
||||||
|
!/operator-authoris|operator-authoriz|by work order|PM checkpoint/i.test(why),
|
||||||
|
'the frozen file may not claim operator authorisation, and coordination metadata belongs in the local plan',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue