merge: wave 1 order 1 (executor) into main
CLAUDE.md command table: each row from its owner (trekplan from order 3, trekexecute from order 1, trekreview from order 5). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
ca6ee8779a
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
28 changed files with 2384 additions and 116 deletions
106
lib/verification/done-line.mjs
Normal file
106
lib/verification/done-line.mjs
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
// lib/verification/done-line.mjs
|
||||
// The done line (P9 (4)). A step is "done" only when this script has written
|
||||
// its line, and it writes the line only when the test command exits 0:
|
||||
//
|
||||
// DONE step <N>: `<command>` exit 0 at <HEAD sha> <ISO time>
|
||||
//
|
||||
// appended to the step's report file. Exit 0 = written, 1 = the command failed
|
||||
// (nothing written), 2 = could not run (nothing written; never green): no
|
||||
// --step, a command outside the test-runner allowlist, a command that cannot
|
||||
// start, a working tree that is not a git repo.
|
||||
//
|
||||
// `--check --step <N> --report <file> [--commit <sha>]`: exit 0 when the report
|
||||
// carries a well-formed line for step N (at exactly that commit, with
|
||||
// --commit), 1 when it does not. /trekexecute writes the line right after the
|
||||
// step's checkpoint and checks it with --commit, so the line is bound to the
|
||||
// commit the test passed on.
|
||||
//
|
||||
// Adapted from obra/superpowers@8ca22db (MIT, (c) 2025 Jesse Vincent),
|
||||
// skills/executing-plans/scripts/task-done: the completion line is appended
|
||||
// only when the test command exits 0.
|
||||
|
||||
import { appendFileSync, existsSync, readFileSync, realpathSync } from 'node:fs';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { allowedCommand, screenCommand } from './criteria-runner.mjs';
|
||||
import { commandEnv, DEFAULT_TIMEOUT_MS, repoRoot } from './red-proof.mjs';
|
||||
|
||||
export function doneLineRe(step, commit) {
|
||||
const sha = commit ? String(commit).replace(/[^0-9a-f]/g, '') : '[0-9a-f]{40}';
|
||||
return new RegExp(`^DONE step ${step}: \`.+\` exit 0 at ${sha} \\d{4}-\\d{2}-\\d{2}T\\S+$`, 'm');
|
||||
}
|
||||
|
||||
/**
|
||||
* With `commit` (the step's checkpoint sha, full 40 hex), the line must name
|
||||
* exactly that commit. A regex cannot tell a typed line from a written one; the
|
||||
* commit binds the line to the state the test ran on (/trekexecute writes the
|
||||
* line right after the checkpoint, so HEAD is the step's own commit).
|
||||
*/
|
||||
export function hasDoneLine(reportPath, step, commit) {
|
||||
if (!existsSync(reportPath)) return false;
|
||||
return doneLineRe(step, commit).test(readFileSync(reportPath, 'utf8'));
|
||||
}
|
||||
|
||||
/** @returns {{status: 'written'|'failed'|'not-run', reason: string, line?: string, exit?: number|null}} */
|
||||
export function writeDoneLine({ cwd, step, cmd, report, timeoutMs = DEFAULT_TIMEOUT_MS }) {
|
||||
if (!/^\d+$/.test(String(step ?? ''))) return { status: 'not-run', reason: '--step must be a step number' };
|
||||
if (!report) return { status: 'not-run', reason: '--report is required' };
|
||||
// The plan's Verify command is relative to the repo root, as in red-proof.mjs.
|
||||
const dir = repoRoot(resolve(cwd ?? process.cwd()));
|
||||
if (!dir) return { status: 'not-run', reason: `${cwd ?? process.cwd()} is not inside a git repository` };
|
||||
const allowed = allowedCommand(cmd, { cwd: dir });
|
||||
if (!allowed.allowed) return { status: 'not-run', reason: `command not run: ${allowed.reason}` };
|
||||
const screen = screenCommand(cmd);
|
||||
if (!screen.allowed) return { status: 'not-run', reason: `command blocked: ${screen.rule}` };
|
||||
const head = spawnSync('git', ['rev-parse', '--verify', '--quiet', 'HEAD'], { cwd: dir, encoding: 'utf8' });
|
||||
if (head.status !== 0) return { status: 'not-run', reason: `no git HEAD in ${dir}` };
|
||||
const r = spawnSync('/bin/sh', ['-c', cmd], { cwd: dir, env: commandEnv(), encoding: 'utf8', timeout: timeoutMs, maxBuffer: 64 * 1024 * 1024 });
|
||||
if (r.error || r.status === null || r.status === 126 || r.status === 127) {
|
||||
return { status: 'not-run', reason: `the command did not run to an exit code (${r.error?.message ?? r.signal ?? `exit ${r.status}`})`, exit: r.status };
|
||||
}
|
||||
if (r.status !== 0) return { status: 'failed', reason: `the test command exited ${r.status}`, exit: r.status };
|
||||
const line = `DONE step ${step}: \`${cmd}\` exit 0 at ${head.stdout.trim()} ${new Date().toISOString()}`;
|
||||
appendFileSync(report, `${line}\n`);
|
||||
return { status: 'written', reason: 'the test command exited 0', line, exit: 0 };
|
||||
}
|
||||
|
||||
const USAGE =
|
||||
'usage: done-line.mjs --step <N> --cmd "<test command>" --report <file> [--cwd <dir>] [--json]\n' +
|
||||
' done-line.mjs --check --step <N> --report <file> [--commit <sha>]';
|
||||
|
||||
export function main(argv) {
|
||||
const o = {};
|
||||
let check = false;
|
||||
let json = false;
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === '--step' && argv[i + 1]) o.step = argv[++i];
|
||||
else if (a === '--cmd' && argv[i + 1]) o.cmd = argv[++i];
|
||||
else if (a === '--report' && argv[i + 1]) o.report = resolve(argv[++i]);
|
||||
else if (a === '--cwd' && argv[i + 1]) o.cwd = resolve(argv[++i]);
|
||||
else if (a === '--timeout' && argv[i + 1]) o.timeoutMs = Number(argv[++i]);
|
||||
else if (a === '--check') check = true;
|
||||
else if (a === '--commit' && argv[i + 1]) o.commit = argv[++i];
|
||||
else if (a === '--json') json = true;
|
||||
else { process.stderr.write(`done-line: unknown argument ${a}\n${USAGE}\n`); return 2; }
|
||||
}
|
||||
if (check) {
|
||||
if (!/^\d+$/.test(String(o.step ?? '')) || !o.report) { process.stderr.write(`done-line: --check needs --step and --report\n${USAGE}\n`); return 2; }
|
||||
const ok = hasDoneLine(o.report, o.step, o.commit);
|
||||
process.stdout.write(json ? `${JSON.stringify({ step: Number(o.step), done: ok })}\n` : `step ${o.step}: ${ok ? 'DONE line present' : 'no DONE line - not done'}\n`);
|
||||
return ok ? 0 : 1;
|
||||
}
|
||||
let r;
|
||||
try { r = writeDoneLine(o); }
|
||||
catch (e) { r = { status: 'not-run', reason: `done-line crashed: ${e.message}` }; }
|
||||
process.stdout.write(json ? `${JSON.stringify(r)}\n` : `done line: ${r.status.toUpperCase()} - ${r.line ?? r.reason}\n`);
|
||||
return r.status === 'written' ? 0 : r.status === 'failed' ? 1 : 2;
|
||||
}
|
||||
|
||||
function isMain() {
|
||||
try { return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)); }
|
||||
catch { return false; }
|
||||
}
|
||||
|
||||
if (process.argv[1] && isMain()) process.exitCode = main(process.argv.slice(2));
|
||||
251
lib/verification/red-proof.mjs
Normal file
251
lib/verification/red-proof.mjs
Normal file
|
|
@ -0,0 +1,251 @@
|
|||
// lib/verification/red-proof.mjs
|
||||
// Proof of red (P9 (1)). A step's test is run by a command:
|
||||
//
|
||||
// 1. against the START POINT (the step's recorded start_sha, with the step's
|
||||
// test files copied in): it must FAIL - otherwise the test proves nothing
|
||||
// about the change;
|
||||
// 2. against the RESULT (the working tree as the step left it): it must PASS;
|
||||
// 3. CONTROL: the start-point copy with the step's whole change laid on top
|
||||
// must PASS too. A copy that cannot go green even with the change failed
|
||||
// step 1 for an environment reason (a missing dependency, a path, a tool),
|
||||
// not because the change was missing - that is "not-run", never proven.
|
||||
//
|
||||
// `--red-only` runs step 1 alone: the gate the step helper passes BEFORE it
|
||||
// builds (and locks the test). The executor's own check afterwards is the full
|
||||
// run, and only the full run is authoritative.
|
||||
//
|
||||
// Exit 0 = proven, 1 = rejected (`not-red-at-base` or `not-green-at-result`),
|
||||
// 2 = could not run (`not-run`: no test files, an unknown start point, a command
|
||||
// outside the test-runner allowlist, a command that cannot start, a control run
|
||||
// that stays red). 2 is never a pass and never "proven".
|
||||
//
|
||||
// The start point is a temp `git clone --shared` of the repo checked out at
|
||||
// start_sha - a real git checkout, so a git-reading command behaves as it does
|
||||
// in the repo, and nothing in the working tree or the repo's worktree list is
|
||||
// touched. Every gitignored top-level entry of the working tree (node_modules,
|
||||
// .venv, a build cache) is symlinked in, so the copy has the same dependencies;
|
||||
// `--link a,b` adds deeper ones. Measured 2026-10-06 (review): with a bare
|
||||
// `git archive` copy, a test that needs node_modules "proved red" with no change.
|
||||
// Known limit, failing safe: a dependency that points back INTO the working
|
||||
// tree (an editable `pip install -e .`, a workspace symlink) makes the start
|
||||
// point see the new code, and a correct step reads `not-red-at-base`.
|
||||
//
|
||||
// Everything runs at the root of --cwd's repository: plan paths and commands
|
||||
// are root-relative, and `git archive`/`clone` paths from a subdirectory are
|
||||
// not (measured 2026-10-06).
|
||||
//
|
||||
// The command must be ONE test-runner invocation without shell operators
|
||||
// (criteria-runner.mjs `allowedCommand`), and it is screened through the
|
||||
// plugin's own PreToolUse denylist (`screenCommand`): a command spawned from
|
||||
// node never passes through the Bash tool, so the hook cannot fire by itself.
|
||||
//
|
||||
// When the output is TAP (`--test-reporter=tap`, pytest-tap, ...), the counts
|
||||
// `# tests N` / `# fail N` are reported per run. They show WHAT the red was:
|
||||
// "fail 1 of 1" on a node test file is usually a load error (an import the
|
||||
// start point lacks), not named failing tests - see
|
||||
// tests/fixtures/red-first/02243c6-always-allow-shim.mjs, re-measured through
|
||||
// this tool in tests/lib/red-proof.test.mjs.
|
||||
//
|
||||
// Borrowed: GitHub Spec Kit (MIT, github/spec-kit@2e941fe) - its test-first
|
||||
// rule is an instruction there; here a command proves it.
|
||||
|
||||
import { cpSync, existsSync, lstatSync, mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync } from 'node:fs';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, isAbsolute, join, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { allowedCommand, screenCommand } from './criteria-runner.mjs';
|
||||
|
||||
export const DEFAULT_TIMEOUT_MS = 900_000;
|
||||
const TAIL = 2000;
|
||||
|
||||
function tap(output) {
|
||||
const tests = String(output).match(/^# tests (\d+)$/m);
|
||||
const fail = String(output).match(/^# fail (\d+)$/m);
|
||||
return tests && fail ? { tests: Number(tests[1]), fail: Number(fail[1]) } : null;
|
||||
}
|
||||
|
||||
// The command's exit code must be its own. Under `node --test` a child inherits
|
||||
// NODE_TEST_CONTEXT, and a nested `node --test` then reports to that parent and
|
||||
// exits 0 whatever its tests did (measured 2026-10-06: a red start point read as
|
||||
// green). So the variable never reaches the command.
|
||||
export function commandEnv(env = process.env) {
|
||||
const { NODE_TEST_CONTEXT, ...rest } = env;
|
||||
return rest;
|
||||
}
|
||||
|
||||
export function repoRoot(dir) {
|
||||
const top = spawnSync('git', ['rev-parse', '--show-toplevel'], { cwd: dir, encoding: 'utf8' });
|
||||
return top.status === 0 ? top.stdout.trim() : null;
|
||||
}
|
||||
|
||||
function run(cmd, cwd, timeoutMs) {
|
||||
const r = spawnSync('/bin/sh', ['-c', cmd], { cwd, env: commandEnv(), encoding: 'utf8', timeout: timeoutMs, maxBuffer: 64 * 1024 * 1024 });
|
||||
const output = `${r.stdout ?? ''}${r.stderr ?? ''}`;
|
||||
return { exit: r.status, signal: r.signal ?? null, tap: tap(output), tail: output.slice(-TAIL), spawnError: r.error ? r.error.message : null };
|
||||
}
|
||||
|
||||
const notRun = (reason, extra = {}) => ({ status: 'not-run', reason, ...extra });
|
||||
|
||||
// A shell that could not find or execute the command (126/127), a signal (the
|
||||
// timeout), or a spawn error is not a test verdict at all.
|
||||
function unrunnable(r) {
|
||||
if (r.spawnError) return `the command could not start: ${r.spawnError}`;
|
||||
if (r.exit === null) return `the command was killed (${r.signal ?? 'timeout'})`;
|
||||
if (r.exit === 126 || r.exit === 127) return `the shell could not run the command (exit ${r.exit})`;
|
||||
return null;
|
||||
}
|
||||
|
||||
const git = (cwd, args) => spawnSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 });
|
||||
const lines = (s) => String(s ?? '').split('\n').filter(Boolean);
|
||||
|
||||
function safeRel(p) {
|
||||
return p && !isAbsolute(p) && !p.split('/').includes('..');
|
||||
}
|
||||
|
||||
function copyIn(from, to, rel) {
|
||||
mkdirSync(dirname(join(to, rel)), { recursive: true });
|
||||
rmSync(join(to, rel), { recursive: true, force: true });
|
||||
cpSync(join(from, rel), join(to, rel), { recursive: true, verbatimSymlinks: true });
|
||||
}
|
||||
|
||||
/** Symlink every gitignored top-level entry of the working tree, plus `extra`, into `tree`. */
|
||||
function linkDependencies(root, tree, extra) {
|
||||
const linked = [];
|
||||
const ignored = lines(git(root, ['ls-files', '--others', '--ignored', '--exclude-standard', '--directory']).stdout)
|
||||
.map((p) => p.replace(/\/$/, '').split('/')[0]);
|
||||
for (const l of new Set([...ignored, ...extra])) {
|
||||
if (!safeRel(l) || l === '.git') continue;
|
||||
if (existsSync(join(root, l)) && !existsSync(join(tree, l))) {
|
||||
mkdirSync(dirname(join(tree, l)), { recursive: true });
|
||||
symlinkSync(join(root, l), join(tree, l));
|
||||
linked.push(l);
|
||||
}
|
||||
}
|
||||
return linked;
|
||||
}
|
||||
|
||||
/** Lay the working tree's change since `base` onto `tree` (tracked edits, deletions, untracked files). */
|
||||
function applyResult(root, tree, base) {
|
||||
const status = lines(git(root, ['diff', '--name-status', '--no-renames', base]).stdout);
|
||||
for (const s of status) {
|
||||
const [code, rel] = s.split('\t');
|
||||
if (!safeRel(rel)) continue;
|
||||
if (code.startsWith('D')) rmSync(join(tree, rel), { recursive: true, force: true });
|
||||
else if (existsSync(join(root, rel)) || lstatSync(join(root, rel), { throwIfNoEntry: false })) copyIn(root, tree, rel);
|
||||
}
|
||||
for (const rel of lines(git(root, ['ls-files', '--others', '--exclude-standard']).stdout)) {
|
||||
if (safeRel(rel)) copyIn(root, tree, rel);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {{cwd: string, base: string, cmd: string, tests: string[], link?: string[], redOnly?: boolean, timeoutMs?: number}} o
|
||||
* @returns {{status: 'proven'|'rejected'|'not-run', reason: string, base?: object, result?: object, control?: object}}
|
||||
*/
|
||||
export function proveRed(o) {
|
||||
const given = resolve(o.cwd ?? process.cwd());
|
||||
const timeoutMs = o.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
||||
const tests = (o.tests ?? []).filter(Boolean);
|
||||
if (!existsSync(given)) return notRun(`--cwd ${given} does not exist`);
|
||||
const cwd = repoRoot(given);
|
||||
if (!cwd) return notRun(`--cwd ${given} is not inside a git repository`);
|
||||
if (tests.length === 0) return notRun('no test files given: a step without its test cannot prove red');
|
||||
for (const t of tests) {
|
||||
if (!safeRel(t)) return notRun(`test path must be repo-relative: ${t}`);
|
||||
if (!existsSync(join(cwd, t))) return notRun(`test file not found in the working tree: ${t}`);
|
||||
}
|
||||
for (const l of o.link ?? []) if (!safeRel(l)) return notRun(`--link must be repo-relative: ${l}`);
|
||||
const allowed = allowedCommand(o.cmd, { cwd });
|
||||
if (!allowed.allowed) return notRun(`command not run: ${allowed.reason}`);
|
||||
const screen = screenCommand(o.cmd);
|
||||
if (!screen.allowed) return notRun(`command blocked: ${screen.rule}`);
|
||||
const sha = git(cwd, ['rev-parse', '--verify', '--quiet', `${o.base}^{commit}`]);
|
||||
if (sha.status !== 0) return notRun(`cannot resolve the start point ${o.base} in ${cwd}`);
|
||||
const baseSha = sha.stdout.trim();
|
||||
|
||||
// The result first: it is the working tree as it stands. `redOnly` is the
|
||||
// gate before building (the step helper's own check): the result is not run.
|
||||
let result;
|
||||
if (!o.redOnly) {
|
||||
result = run(o.cmd, cwd, timeoutMs);
|
||||
const resultBroken = unrunnable(result);
|
||||
if (resultBroken) return notRun(`at the result: ${resultBroken}`, { result });
|
||||
}
|
||||
|
||||
const tmp = mkdtempSync(join(tmpdir(), 'voyage-red-proof-'));
|
||||
try {
|
||||
const tree = join(tmp, 'tree');
|
||||
const c = git(tmp, ['clone', '--quiet', '--shared', '--no-checkout', cwd, tree]);
|
||||
if (c.status !== 0) return notRun(`could not clone the start point: ${String(c.stderr).trim()}`, { result });
|
||||
const co = git(tree, ['checkout', '--quiet', '--detach', baseSha]);
|
||||
if (co.status !== 0) return notRun(`could not check out ${baseSha}: ${String(co.stderr).trim()}`, { result });
|
||||
for (const t of tests) copyIn(cwd, tree, t);
|
||||
const linked = linkDependencies(cwd, tree, o.link ?? []);
|
||||
|
||||
const base = run(o.cmd, tree, timeoutMs);
|
||||
const baseBroken = unrunnable(base);
|
||||
if (baseBroken) return notRun(`at the start point: ${baseBroken}`, { base, result, linked });
|
||||
const out = { base_sha: baseSha, linked, base, result };
|
||||
if (base.exit === 0) return { status: 'rejected', reason: 'not-red-at-base', ...out };
|
||||
if (o.redOnly) return { status: 'proven', reason: 'red at the start point (--red-only: the result was not run)', base_sha: baseSha, linked, base };
|
||||
if (result.exit !== 0) return { status: 'rejected', reason: 'not-green-at-result', ...out };
|
||||
|
||||
// Control: the same copy with the whole change laid on. Red here means the
|
||||
// copy itself cannot run the test green, so its red proved nothing.
|
||||
applyResult(cwd, tree, baseSha);
|
||||
const control = run(o.cmd, tree, timeoutMs);
|
||||
if (control.exit !== 0) {
|
||||
return notRun('the control run (start-point copy + the change) stayed red: the start point failed for an environment reason, not for the missing change', { ...out, control });
|
||||
}
|
||||
return { status: 'proven', reason: 'red at the start point, green at the result, green in the control', ...out, control };
|
||||
} finally {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
// --- CLI ------------------------------------------------------------------------
|
||||
|
||||
const USAGE =
|
||||
'usage: red-proof.mjs --base <start_sha> --cmd "<test command>" --tests <a,b> [--red-only] [--cwd <dir>] [--link <dir,...>] [--timeout <ms>] [--json]';
|
||||
|
||||
const list = (v) => String(v).split(',').map((s) => s.trim()).filter(Boolean);
|
||||
|
||||
export function main(argv) {
|
||||
const o = { cwd: process.cwd(), tests: [], link: [] };
|
||||
let json = false;
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === '--base' && argv[i + 1]) o.base = argv[++i];
|
||||
else if (a === '--cmd' && argv[i + 1]) o.cmd = argv[++i];
|
||||
else if (a === '--tests' && argv[i + 1]) o.tests = list(argv[++i]);
|
||||
else if (a === '--cwd' && argv[i + 1]) o.cwd = resolve(argv[++i]);
|
||||
else if (a === '--link' && argv[i + 1]) o.link = list(argv[++i]);
|
||||
else if (a === '--red-only') o.redOnly = true;
|
||||
else if (a === '--timeout' && argv[i + 1]) o.timeoutMs = Number(argv[++i]);
|
||||
else if (a === '--json') json = true;
|
||||
else { process.stderr.write(`red-proof: unknown argument ${a} (--tests takes ONE comma-separated list)\n${USAGE}\n`); return 2; }
|
||||
}
|
||||
if (!o.base || !o.cmd) { process.stderr.write(`red-proof: --base and --cmd are required\n${USAGE}\n`); return 2; }
|
||||
if (o.timeoutMs !== undefined && !Number.isFinite(o.timeoutMs)) { process.stderr.write(`red-proof: --timeout must be a number\n${USAGE}\n`); return 2; }
|
||||
let r;
|
||||
try { r = proveRed(o); }
|
||||
catch (e) { r = notRun(`red-proof crashed: ${e.message}`); }
|
||||
if (json) process.stdout.write(JSON.stringify(r, null, 2) + '\n');
|
||||
else {
|
||||
const counts = (x) => (x?.tap ? ` (tap: ${x.tap.fail} of ${x.tap.tests} failed)` : '');
|
||||
process.stdout.write(`red proof: ${r.status.toUpperCase()} - ${r.reason}\n`);
|
||||
if (r.base) process.stdout.write(` start point ${r.base_sha ?? o.base}: exit ${r.base.exit}${counts(r.base)}\n`);
|
||||
if (r.result) process.stdout.write(` result: exit ${r.result.exit}${counts(r.result)}\n`);
|
||||
if (r.control) process.stdout.write(` control: exit ${r.control.exit}${counts(r.control)}\n`);
|
||||
if (r.linked?.length) process.stdout.write(` linked into the start point: ${r.linked.join(', ')}\n`);
|
||||
}
|
||||
return r.status === 'proven' ? 0 : r.status === 'rejected' ? 1 : 2;
|
||||
}
|
||||
|
||||
function isMain() {
|
||||
try { return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)); }
|
||||
catch { return false; }
|
||||
}
|
||||
|
||||
if (process.argv[1] && isMain()) process.exitCode = main(process.argv.slice(2));
|
||||
290
lib/verification/test-integrity.mjs
Normal file
290
lib/verification/test-integrity.mjs
Normal file
|
|
@ -0,0 +1,290 @@
|
|||
// lib/verification/test-integrity.mjs
|
||||
// The floor guard (P9 (2)): read a step's diff and report every change that
|
||||
// makes the tests weaker instead of the code better.
|
||||
//
|
||||
// removed-assert a test file loses more assertions than it gains
|
||||
// deleted-test a test file loses more test declarations than it gains
|
||||
// skipped-test a test is skipped, marked todo/xfail, or focused (.only)
|
||||
// muted-check a check is silenced: `|| true`, continue-on-error,
|
||||
// --passWithNoTests, an inline lint/type suppression ...
|
||||
// stub non-test code gains a "not implemented" body
|
||||
// loosened-threshold a numeric bound in a test or a check config moves in the
|
||||
// permissive direction (`ms < 200` -> `ms < 2000`)
|
||||
//
|
||||
// Exit 0 = clean, 1 = findings, 2 = could not run (no input, unreadable input,
|
||||
// an unresolvable --base). 2 is never green: an absent measurement is reported
|
||||
// as absent, not as a pass.
|
||||
//
|
||||
// A finding is not a verdict. /trekexecute hands every finding to its
|
||||
// controller, which either quotes the plan line that asked for the change or
|
||||
// fails the step; nothing is dropped silently.
|
||||
//
|
||||
// Borrowed: the floor guard of addyosmani/agent-skills@9d0c60d (MIT),
|
||||
// constraint-driven development - mechanized here as a diff reader so a
|
||||
// command, not the model's word, says whether a test was weakened.
|
||||
|
||||
import { readFileSync, realpathSync } from 'node:fs';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
// --- what counts as a test / a check config ----------------------------------
|
||||
|
||||
const TEST_PATH_RE = [
|
||||
/(^|\/)(tests?|__tests__|spec|specs)\//,
|
||||
/\.(test|spec)\.[a-z0-9]+$/i,
|
||||
/_(test|spec)\.[a-z0-9]+$/i,
|
||||
/(^|\/)test_[^/]*\.py$/,
|
||||
];
|
||||
|
||||
const CHECK_CONFIG_RE =
|
||||
/(^|\/)(\.github|\.forgejo|\.gitea|\.woodpecker)\/|(^|\/)(\.gitlab-ci\.yml|Makefile|package\.json|pyproject\.toml|setup\.cfg|tox\.ini|\.pre-commit-config\.yaml|tsconfig[^/]*\.json|\.eslintrc[^/]*|eslint\.config\.[a-z]+|(jest|vitest|playwright)\.config\.[a-z]+|\.c8rc[^/]*|\.nycrc[^/]*|codecov\.ya?ml)$/;
|
||||
|
||||
export function isTestPath(p) {
|
||||
return TEST_PATH_RE.some((re) => re.test(p));
|
||||
}
|
||||
|
||||
const isCheckPath = (p) => isTestPath(p) || CHECK_CONFIG_RE.test(p);
|
||||
const PROSE_RE = /\.(md|markdown|mdx|txt|rst|adoc)$/i;
|
||||
|
||||
// --- the patterns -------------------------------------------------------------
|
||||
|
||||
const ASSERT_RE =
|
||||
/\bassert\w*\b|\bexpect\s*\(|\.should\b|\bt\.(equal|is|ok|true|false|deepEqual|throws|notThrows|match)\b|\bself\.assert\w+|\brequire\.\w+\(|\bt\.(Error|Fatal)f?\(/g;
|
||||
|
||||
const TEST_DECL_RE =
|
||||
/^\s*(test|it|describe|suite)(\.\w+)?\s*\(|^\s*(async\s+)?def\s+test_\w*|^\s*func\s+Test\w*|^\s*#\[test\]|@Test\b/;
|
||||
|
||||
const SKIP_RE =
|
||||
/\b(test|it|describe|suite)\.(skip|skipIf|todo|only)\s*\(|\bt\.(skip|todo)\s*\(|\bx(it|describe|test)\s*\(|\bf(it|describe)\s*\(|\{\s*(skip|todo)\s*:\s*(true|['"`])|@pytest\.mark\.(skip|skipif|xfail)\b|\bpytest\.skip\s*\(|@unittest\.skip|\bt\.Skip(Now|f)?\s*\(|#\[ignore\]|@(Disabled|Ignore)\b/;
|
||||
|
||||
// Inline suppressions silence a check wherever they appear.
|
||||
const SUPPRESS_RE =
|
||||
/@ts-ignore|@ts-nocheck|eslint-disable|#\s*type:\s*ignore|#\s*noqa\b|\/\/\s*nolint\b|#\s*pragma:\s*no\s*cover|istanbul\s+ignore|c8\s+ignore/;
|
||||
|
||||
// Exit-code mutes count only where a check runs: tests, scripts under tests/, CI
|
||||
// and check configs. In ordinary code `|| true` is a normal shell idiom.
|
||||
const MUTE_RE =
|
||||
/\|\|\s*true\b|\|\|\s*:\s*($|[;)])|\|\|\s*exit\s+0\b|continue-on-error\s*:\s*true|--passWithNoTests|--pass-with-no-tests|\bset\s+\+e\b|--no-verify\b|allow_failure\s*:\s*true/;
|
||||
|
||||
const STUB_RE =
|
||||
/throw\s+new\s+Error\s*\(\s*['"`]\s*(not\s+implemented|unimplemented|todo)\b|\braise\s+NotImplementedError\b|\b(unimplemented|todo)!\s*\(|panic\s*\(\s*"(not\s+implemented|todo)|(\/\/|#)\s*stub\b|TODO:?\s*implement/i;
|
||||
|
||||
// Number after a "smaller is stricter" bound: bigger = looser.
|
||||
const UPPER_BOUND_RE = /(<=?|toBeLessThan(OrEqual)?|lessThan|\bmax\w*|timeout\w*|tolerance|epsilon|delta|within|retries)\W*$/i;
|
||||
// Number after a "bigger is stricter" bound: smaller = looser.
|
||||
const LOWER_BOUND_RE = /(>=?|toBeGreaterThan(OrEqual)?|greaterThan|\bmin\w*|threshold|coverage|branches|lines|functions|statements)\W*$/i;
|
||||
|
||||
const NUM_RE = /\d+(?:\.\d+)?/g;
|
||||
|
||||
// --- diff parsing ---------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Parse a unified diff into files with hunks of {type: ' '|'-'|'+', text, line}.
|
||||
* Returns null when non-empty input carries no diff header at all.
|
||||
*/
|
||||
export function parseDiff(text) {
|
||||
const src = String(text ?? '');
|
||||
if (src.trim() === '') return [];
|
||||
if (!/^(diff --git |--- |\+\+\+ )/m.test(src)) return null;
|
||||
const files = [];
|
||||
let file = null;
|
||||
let hunk = null;
|
||||
let oldLine = 0;
|
||||
let newLine = 0;
|
||||
// Lines still owed to the current hunk, from its @@ header. While a hunk is
|
||||
// owed lines, every line is content - even one that reads "--- x" or
|
||||
// "+++ y" (a removed "-- x" / an added "++ y"). Headers only between hunks.
|
||||
let oldLeft = 0;
|
||||
let newLeft = 0;
|
||||
const lines = src.split('\n');
|
||||
for (const l of lines) {
|
||||
if (hunk && (oldLeft > 0 || newLeft > 0)) {
|
||||
if (l.startsWith('\\')) continue; // "\ No newline at end of file"
|
||||
if (l.startsWith('-')) { hunk.push({ type: '-', text: l.slice(1), line: oldLine++ }); oldLeft--; continue; }
|
||||
if (l.startsWith('+')) { hunk.push({ type: '+', text: l.slice(1), line: newLine++ }); newLeft--; continue; }
|
||||
if (l.startsWith(' ') || l === '') {
|
||||
hunk.push({ type: ' ', text: l.slice(1), line: newLine });
|
||||
oldLine++; newLine++; oldLeft--; newLeft--;
|
||||
continue;
|
||||
}
|
||||
// Anything else ends a malformed hunk early; fall through to headers.
|
||||
oldLeft = 0; newLeft = 0;
|
||||
}
|
||||
if (l.startsWith('diff --git ')) {
|
||||
const m = l.match(/^diff --git a\/(.+) b\/(.+)$/);
|
||||
file = { path: m ? m[2] : l.slice(11), deleted: false, hunks: [] };
|
||||
files.push(file);
|
||||
hunk = null;
|
||||
continue;
|
||||
}
|
||||
if (l.startsWith('--- ')) {
|
||||
if (!file || file.hunks.length > 0) { file = { path: '', deleted: false, hunks: [] }; files.push(file); }
|
||||
const oldPath = l.slice(4).replace(/^a\//, '');
|
||||
if (!file.path && oldPath !== '/dev/null') file.path = oldPath;
|
||||
hunk = null;
|
||||
continue;
|
||||
}
|
||||
if (l.startsWith('+++ ') && file) {
|
||||
const newPath = l.slice(4).replace(/^b\//, '');
|
||||
if (newPath === '/dev/null') file.deleted = true;
|
||||
else file.path = newPath;
|
||||
continue;
|
||||
}
|
||||
const h = l.match(/^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/);
|
||||
if (h && file) {
|
||||
oldLine = Number(h[1]);
|
||||
newLine = Number(h[3]);
|
||||
oldLeft = h[2] === undefined ? 1 : Number(h[2]);
|
||||
newLeft = h[4] === undefined ? 1 : Number(h[4]);
|
||||
hunk = [];
|
||||
file.hunks.push(hunk);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
// --- detectors ------------------------------------------------------------------
|
||||
|
||||
const count = (lines, re) => lines.reduce((n, l) => n + (l.text.match(new RegExp(re.source, 'g')) || []).length, 0);
|
||||
const norm = (s) => s.trim().replace(/\s+/g, ' ');
|
||||
|
||||
function addedNotRemoved(removed, added, re) {
|
||||
const before = new Set(removed.filter((l) => re.test(l.text)).map((l) => norm(l.text)));
|
||||
return added.filter((l) => re.test(l.text) && !before.has(norm(l.text)));
|
||||
}
|
||||
|
||||
/** A paired -/+ line whose only change is numbers; returns the looser numbers. */
|
||||
function loosened(oldText, newText) {
|
||||
const shape = (s) => norm(s).replace(NUM_RE, '#');
|
||||
if (shape(oldText) !== shape(newText)) return null;
|
||||
const a = [...norm(oldText).matchAll(NUM_RE)];
|
||||
const b = [...norm(newText).matchAll(NUM_RE)];
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
const before = Number(a[i][0]);
|
||||
const after = Number(b[i][0]);
|
||||
if (before === after) continue;
|
||||
const lead = norm(newText).slice(Math.max(0, b[i].index - 30), b[i].index);
|
||||
if (UPPER_BOUND_RE.test(lead) && after > before) return `${before} -> ${after}`;
|
||||
if (LOWER_BOUND_RE.test(lead) && after < before) return `${before} -> ${after}`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function finding(kind, file, l, detail) {
|
||||
return { kind, file, line: l ? l.line : null, text: l ? l.text.trim() : '', ...(detail ? { detail } : {}) };
|
||||
}
|
||||
|
||||
export function analyzeDiff(text) {
|
||||
const files = parseDiff(text);
|
||||
if (files === null) return { status: 'unreadable', findings: [], reason: 'the input is not a unified diff' };
|
||||
const findings = [];
|
||||
for (const f of files) {
|
||||
const all = f.hunks.flat();
|
||||
const removed = all.filter((l) => l.type === '-');
|
||||
const added = all.filter((l) => l.type === '+');
|
||||
const test = isTestPath(f.path);
|
||||
// Prose (docs, notes) describes code; a "not implemented" or an
|
||||
// eslint-disable written in it is text, not a stub or a muted check.
|
||||
const prose = PROSE_RE.test(f.path);
|
||||
|
||||
if (test) {
|
||||
const lostAsserts = count(removed, ASSERT_RE) - count(added, ASSERT_RE);
|
||||
if (lostAsserts > 0) {
|
||||
findings.push(finding('removed-assert', f.path, removed.find((l) => new RegExp(ASSERT_RE.source).test(l.text)), `${lostAsserts} fewer`));
|
||||
}
|
||||
const lostTests = removed.filter((l) => TEST_DECL_RE.test(l.text)).length - added.filter((l) => TEST_DECL_RE.test(l.text)).length;
|
||||
if (lostTests > 0 || (f.deleted && removed.some((l) => TEST_DECL_RE.test(l.text)))) {
|
||||
findings.push(finding('deleted-test', f.path, removed.find((l) => TEST_DECL_RE.test(l.text)), f.deleted ? 'test file deleted' : `${lostTests} fewer`));
|
||||
}
|
||||
for (const l of addedNotRemoved(removed, added, SKIP_RE)) findings.push(finding('skipped-test', f.path, l));
|
||||
}
|
||||
|
||||
if (!prose) for (const l of addedNotRemoved(removed, added, SUPPRESS_RE)) findings.push(finding('muted-check', f.path, l));
|
||||
if (isCheckPath(f.path)) {
|
||||
for (const l of addedNotRemoved(removed, added, MUTE_RE)) findings.push(finding('muted-check', f.path, l));
|
||||
for (const h of f.hunks) {
|
||||
const minus = h.filter((l) => l.type === '-');
|
||||
const plus = h.filter((l) => l.type === '+');
|
||||
for (const m of minus) {
|
||||
for (const p of plus) {
|
||||
const d = loosened(m.text, p.text);
|
||||
if (d) { findings.push(finding('loosened-threshold', f.path, p, d)); break; }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!test && !prose) for (const l of addedNotRemoved(removed, added, STUB_RE)) findings.push(finding('stub', f.path, l));
|
||||
}
|
||||
return { status: findings.length ? 'findings' : 'clean', findings };
|
||||
}
|
||||
|
||||
// --- CLI ------------------------------------------------------------------------
|
||||
|
||||
const USAGE = 'usage: test-integrity.mjs (--diff <file|-> | --base <ref> [--head <ref>]) [--cwd <dir>] [--json]';
|
||||
|
||||
function diffFromGit(cwd, base, head) {
|
||||
const ok = spawnSync('git', ['rev-parse', '--verify', '--quiet', `${base}^{commit}`], { cwd, encoding: 'utf8' });
|
||||
if (ok.status !== 0) return { error: `cannot resolve --base ${base} in ${cwd}` };
|
||||
if (head) {
|
||||
const okHead = spawnSync('git', ['rev-parse', '--verify', '--quiet', `${head}^{commit}`], { cwd, encoding: 'utf8' });
|
||||
if (okHead.status !== 0) return { error: `cannot resolve --head ${head} in ${cwd}` };
|
||||
}
|
||||
// --no-renames: a rename is read as a deletion plus an addition. Paired as a
|
||||
// rename, `foo.test.js -> foo.test.js.bak` showed no removed line at all and
|
||||
// switched a test off silently (review, 2026-10-06).
|
||||
const args = ['diff', '--no-color', '--no-ext-diff', '--no-renames', base, ...(head ? [head] : [])];
|
||||
const r = spawnSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 });
|
||||
if (r.status !== 0) return { error: `git diff failed: ${String(r.stderr).trim()}` };
|
||||
return { text: r.stdout };
|
||||
}
|
||||
|
||||
export function main(argv) {
|
||||
let diffPath = null;
|
||||
let base = null;
|
||||
let head = null;
|
||||
let cwd = process.cwd();
|
||||
let json = false;
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === '--diff' && argv[i + 1] !== undefined) diffPath = argv[++i];
|
||||
else if (a === '--base' && argv[i + 1]) base = argv[++i];
|
||||
else if (a === '--head' && argv[i + 1]) head = argv[++i];
|
||||
else if (a === '--cwd' && argv[i + 1]) cwd = resolve(argv[++i]);
|
||||
else if (a === '--json') json = true;
|
||||
else { process.stderr.write(`test-integrity: unknown argument ${a}\n${USAGE}\n`); return 2; }
|
||||
}
|
||||
if ((diffPath === null) === (base === null)) {
|
||||
process.stderr.write(`test-integrity: give exactly one of --diff or --base\n${USAGE}\n`);
|
||||
return 2;
|
||||
}
|
||||
let text;
|
||||
if (diffPath !== null) {
|
||||
try { text = readFileSync(diffPath === '-' ? 0 : diffPath, 'utf8'); }
|
||||
catch (e) { return out({ status: 'unreadable', findings: [], reason: `cannot read ${diffPath}: ${e.message}` }, json); }
|
||||
} else {
|
||||
const g = diffFromGit(cwd, base, head);
|
||||
if (g.error) return out({ status: 'unreadable', findings: [], reason: g.error }, json);
|
||||
text = g.text;
|
||||
}
|
||||
return out(analyzeDiff(text), json);
|
||||
}
|
||||
|
||||
function out(r, json) {
|
||||
if (json) process.stdout.write(JSON.stringify(r, null, 2) + '\n');
|
||||
else if (r.status === 'unreadable') process.stdout.write(`floor guard: NOT RUN - ${r.reason} (exit 2, never green)\n`);
|
||||
else if (r.status === 'clean') process.stdout.write('floor guard: clean\n');
|
||||
else {
|
||||
process.stdout.write(`floor guard: ${r.findings.length} finding(s)\n`);
|
||||
for (const f of r.findings) process.stdout.write(` ${f.kind} ${f.file}:${f.line ?? '-'} ${f.text}${f.detail ? ` (${f.detail})` : ''}\n`);
|
||||
}
|
||||
return r.status === 'clean' ? 0 : r.status === 'findings' ? 1 : 2;
|
||||
}
|
||||
|
||||
function isMain() {
|
||||
try { return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)); }
|
||||
catch { return false; }
|
||||
}
|
||||
|
||||
if (process.argv[1] && isMain()) process.exitCode = main(process.argv.slice(2));
|
||||
Loading…
Add table
Add a link
Reference in a new issue