feat(trekplan): Regression and On failure step rules, and trial 2's minor fixes
The minor points (B8-B12, C13, C14, C16, C17) of the second voyage-lekerepo trial (2026-10-08), each verified against the code and started with a test that failed first. - K10: optional step field `Regression:` - one command the criteria runner allows, unpiped; plan-validator refuses one that can never run (PLAN_STEP_REGRESSION_NOT_RUNNABLE). Any plan_version, no bump. - K11: `On failure:` opens with revert | retry | skip | escalate, or the validator warns (PLAN_STEP_ON_FAILURE_UNKNOWN); onFailureAction in plan-schema. - B8 (operator decision 2026-10-08): minors open after the last critic round are listed numbered in the Phase 10 summary; the ones the operator picks are fixed and get one more round (--extra-round when the 3 are spent). - B11: plan-review dedup merges two reviewers' findings on the same file:line with the same content (trial pair: Jaccard 0.44, overlap 0.83); the Jaccard threshold for every other pair is unchanged. - B12: /trekbrief Step 4c says the reviewer runs in the background and is waited for, not "foreground, blocking". - C13: the Phase 4 size count includes .mjs, .cjs, .mts, .cts. - C14: --evidence shows the first output line on PASS rows. - C16: findings-schema checks path:line citations in finding prose (error) and in the prose around the block (warning) against the file. - C17: git-historian gets the brief, never proposes changing a frozen file, and names the remote from `git remote -v`. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
4ee711a6ac
commit
255b6b4bf8
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
15 changed files with 675 additions and 24 deletions
|
|
@ -263,6 +263,24 @@ export function parseStepFields(body) {
|
|||
return fields;
|
||||
}
|
||||
|
||||
// ---- `On failure:` (K11) ---------------------------------------------------
|
||||
|
||||
// The recovery words /trekexecute reads from a step's `On failure:` ("Parse
|
||||
// steps"). Free text that opens with none of them left the executor to guess:
|
||||
// "retry once; if still failing, revert ... and STOP" met a ladder of fix rounds
|
||||
// (voyage-lekerepo trial 2, 2026-10-08).
|
||||
export const ON_FAILURE_ACTIONS = Object.freeze(['revert', 'retry', 'skip', 'escalate']);
|
||||
|
||||
/**
|
||||
* The recovery word an `On failure:` text opens with, or null. Case-insensitive;
|
||||
* leading markdown (`**`, backticks, a sub-bullet's `-`) is stripped.
|
||||
*/
|
||||
export function onFailureAction(text) {
|
||||
const s = String(text ?? '').trim().replace(/^[*_`\s-]+/, '').toLowerCase();
|
||||
const m = s.match(/^([a-z]+)\b/);
|
||||
return m && ON_FAILURE_ACTIONS.includes(m[1]) ? m[1] : null;
|
||||
}
|
||||
|
||||
// ---- sections (fence-aware) ------------------------------------------------
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -134,9 +134,50 @@ function checkLineRange(finding, index, repoRoot, errors) {
|
|||
));
|
||||
}
|
||||
|
||||
// The same check for a `path:line` (or `path:a-b`) a reviewer writes in prose:
|
||||
// a finding's title/detail/recommended_action/brief_ref, and the text around
|
||||
// the JSON block. code-correctness-reviewer wrote "`tests/contract.sh:119`
|
||||
// relies on git" about a 33-line file, and nothing looked (voyage-lekerepo
|
||||
// trial 2, 2026-10-08). In a finding's fields a line past the end of a file
|
||||
// that exists is an error (the re-ask can fix the block); in the prose around
|
||||
// the block it is a warning (see validateReviewerOutput). A path absent from
|
||||
// the tree is not judged. The path needs an extension, so `localhost:8080` and
|
||||
// `host.example:443` are no citation of anything in the tree; a path glued to
|
||||
// a `\` or `:` (a Windows path) is not read as one relative to the root.
|
||||
const PROSE_CITE = /(?<![\w./\\:-])((?:\.{0,2}\/)?[\w.-]+(?:\/[\w.-]+)*\.[A-Za-z0-9]+):(\d+)(?:[-–—](\d+))?(?![\w])/g;
|
||||
const PROSE_FIELDS = Object.freeze(['title', 'detail', 'recommended_action', 'brief_ref']);
|
||||
|
||||
function checkProseCitations(text, where, repoRoot, errors, seen) {
|
||||
if (typeof text !== 'string' || text === '') return;
|
||||
PROSE_CITE.lastIndex = 0;
|
||||
let m;
|
||||
while ((m = PROSE_CITE.exec(text)) !== null) {
|
||||
const line = Math.max(Number(m[2]), m[3] === undefined ? 0 : Number(m[3]));
|
||||
if (line <= 0) continue;
|
||||
const n = lineCount(repoRoot, m[1]);
|
||||
if (n === null || line <= n) continue;
|
||||
const key = `${where}\u0000${m[0]}`;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
errors.push(issue(
|
||||
'FINDING_PROSE_LINE_OUT_OF_RANGE',
|
||||
`${where} cites ${m[0]}, but ${m[1]} has ${n} lines`,
|
||||
'Cite the line number in the file itself (Read it), not the line in the diff or patch, or drop the line number.',
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
function checkFindingProse(finding, index, repoRoot, errors, seen) {
|
||||
if (!finding || typeof finding !== 'object') return;
|
||||
for (const field of PROSE_FIELDS) {
|
||||
checkProseCitations(finding[field], `findings[${index}].${field}`, repoRoot, errors, seen);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {unknown} payload
|
||||
* @param {{ repoRoot?: string }} [opts] - with a repo root, cited lines are checked against the files
|
||||
* @param {{ repoRoot?: string }} [opts] - with a repo root, cited lines are checked against the files:
|
||||
* each finding's `file:line`, and every `path:line` in its prose fields
|
||||
*/
|
||||
export function validateFindings(payload, opts = {}) {
|
||||
if (payload === null || typeof payload !== 'object' || Array.isArray(payload)) {
|
||||
|
|
@ -155,9 +196,13 @@ export function validateFindings(payload, opts = {}) {
|
|||
return { valid: false, errors, warnings, parsed: payload };
|
||||
}
|
||||
|
||||
const seen = new Set();
|
||||
for (let i = 0; i < payload.findings.length; i++) {
|
||||
validateFinding(payload.findings[i], i, errors);
|
||||
if (opts.repoRoot) checkLineRange(payload.findings[i], i, opts.repoRoot, errors);
|
||||
if (opts.repoRoot) {
|
||||
checkLineRange(payload.findings[i], i, opts.repoRoot, errors);
|
||||
checkFindingProse(payload.findings[i], i, opts.repoRoot, errors, seen);
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: errors.length === 0, errors, warnings, parsed: payload };
|
||||
|
|
@ -185,7 +230,17 @@ export function validateReviewerOutput(rawText, opts = {}) {
|
|||
} catch (e) {
|
||||
return fail(issue('FINDINGS_PARSE_ERROR', `Reviewer JSON block did not parse: ${e.message}`));
|
||||
}
|
||||
return validateFindings(parsed, opts);
|
||||
const r = validateFindings(parsed, opts);
|
||||
if (!opts.repoRoot) return r;
|
||||
// The prose around the block (the text outside the LAST json fence; the
|
||||
// block itself was checked field by field above) is a WARNING, not an error:
|
||||
// the bounded re-ask asks for the JSON block only, which cannot correct that
|
||||
// prose, and nothing downstream reads it — an error there could only stop a
|
||||
// review over text no one uses (independent review, 2026-10-08).
|
||||
const at = rawText.lastIndexOf(block);
|
||||
const found = [];
|
||||
checkProseCitations(rawText.slice(0, at) + rawText.slice(at + block.length), 'reviewer prose', opts.repoRoot, found, new Set());
|
||||
return found.length === 0 ? r : { ...r, warnings: [...r.warnings, ...found] };
|
||||
}
|
||||
|
||||
// ---- CLI shim ----------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -3,9 +3,17 @@
|
|||
// merges plan-critic + scope-guardian findings into a single deduplicated
|
||||
// stream, preserving provenance (which agent originally raised each finding).
|
||||
//
|
||||
// Two dedup signals:
|
||||
// Three dedup signals:
|
||||
// 1. Exact match — identical computeFindingId(file:line:rule_key) → merge.
|
||||
// 2. Jaccard ≥ 0.7 on text-token sets → merge (catches near-duplicates).
|
||||
// 3. Same anchor — the same file and the same line (≥ 1), and the content
|
||||
// words of the shorter text mostly in the longer one (overlap ≥ the same
|
||||
// 0.7) → merge. Two reviewers naming one defect at one place word it
|
||||
// differently and pick their own rule_key: trial 2's pair on plan.md:248
|
||||
// measured Jaccard 0.44 and stayed two (voyage-lekerepo, 2026-10-08).
|
||||
// Only an anchored pair from two different reviewers, the shorter text
|
||||
// holding ≥ 6 content words, is measured this way; the threshold for
|
||||
// every other pair is unchanged.
|
||||
//
|
||||
// Provenance is preserved on the surviving finding's `raised_by` array.
|
||||
// Severity is kept (normalized to blocker | major | minor, else null) and a
|
||||
|
|
@ -59,6 +67,54 @@ export function tokenize(text) {
|
|||
return text.toLowerCase().split(/\W+/).filter(t => t.length > 0);
|
||||
}
|
||||
|
||||
// Words that carry no content of a finding.
|
||||
const STOP_WORDS = new Set([
|
||||
'a', 'an', 'the', 'and', 'or', 'but', 'of', 'in', 'on', 'at', 'to', 'for', 'with', 'by', 'from', 'as',
|
||||
'is', 'are', 'was', 'were', 'be', 'been', 'it', 'its', 'this', 'that', 'these', 'those', 'only', 'both',
|
||||
'where', 'which', 'while', 'when', 'than', 'then', 'so', 'if', 'also', 'each', 'every', 'all', 'any',
|
||||
'unlike', 'own',
|
||||
]);
|
||||
|
||||
/** Content words of a finding: tokens minus stop words and single letters, a plural -s folded. */
|
||||
export function contentTokens(text) {
|
||||
return tokenize(text)
|
||||
.filter(t => t.length > 1 && !STOP_WORDS.has(t))
|
||||
.map(t => (t.length > 3 && t.endsWith('s') && !t.endsWith('ss') ? t.slice(0, -1) : t));
|
||||
}
|
||||
|
||||
/** |A ∩ B| / min(|A|, |B|): how much of the shorter text the longer one holds. */
|
||||
function overlap(a, b) {
|
||||
const A = new Set(a);
|
||||
const B = new Set(b);
|
||||
if (A.size === 0 || B.size === 0) return 0;
|
||||
let n = 0;
|
||||
for (const x of A) if (B.has(x)) n += 1;
|
||||
return n / Math.min(A.size, B.size);
|
||||
}
|
||||
|
||||
/** Same file and same line ≥ 1: line 0 (file-scoped) and a missing line anchor nothing. */
|
||||
function sameAnchor(a, b) {
|
||||
return typeof a.file === 'string' && a.file !== '' && a.file === b.file &&
|
||||
Number.isInteger(a.line) && a.line > 0 && a.line === b.line;
|
||||
}
|
||||
|
||||
// Overlap measures the shorter text against the longer, so a short text is
|
||||
// held by almost any longer one on its line ("The token is checked." inside
|
||||
// "The token is not checked before the write." scores 1.0). The anchor rule
|
||||
// therefore merges only a pair from two different reviewers — one reviewer's
|
||||
// two findings on one line are two findings by its own count — and only when
|
||||
// the shorter text names enough to be compared (independent review, 2026-10-08).
|
||||
export const ANCHOR_MIN_CONTENT_WORDS = 6;
|
||||
|
||||
function anchorDuplicate(f, s, threshold) {
|
||||
if (!sameAnchor(f, s)) return false;
|
||||
if (f.raised_by.some(a => s.raised_by.includes(a))) return false;
|
||||
const a = new Set(contentTokens(f.text));
|
||||
const b = new Set(contentTokens(s.text));
|
||||
if (Math.min(a.size, b.size) < ANCHOR_MIN_CONTENT_WORDS) return false;
|
||||
return meetsThreshold(overlap([...a], [...b]), threshold);
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a single agent payload into an array of {agent, finding} pairs.
|
||||
* Tolerates missing payload (returns []).
|
||||
|
|
@ -98,7 +154,7 @@ function annotate(finding, agent) {
|
|||
* findings: Array<object>,
|
||||
* sources: Array<{agent: string, present: boolean}>,
|
||||
* dedup_stats: { total_in: number, total_out: number,
|
||||
* exact_id_dups: number, jaccard_dups: number }
|
||||
* exact_id_dups: number, jaccard_dups: number, anchor_dups: number }
|
||||
* }}
|
||||
*/
|
||||
export function dedupFindings(sources, opts = {}) {
|
||||
|
|
@ -126,9 +182,11 @@ export function dedupFindings(sources, opts = {}) {
|
|||
}
|
||||
}
|
||||
|
||||
// Pass 2 — jaccard on text tokens; merge near-duplicates
|
||||
// Pass 2 — jaccard on text tokens; merge near-duplicates. Pass 3 rides in
|
||||
// the same loop: an anchored pair that Jaccard misses is measured by overlap.
|
||||
const survivors = [];
|
||||
let jaccard_dups = 0;
|
||||
let anchor_dups = 0;
|
||||
for (const f of byId.values()) {
|
||||
const tokens = tokenize(f.text);
|
||||
let merged = false;
|
||||
|
|
@ -140,6 +198,12 @@ export function dedupFindings(sources, opts = {}) {
|
|||
merged = true;
|
||||
break;
|
||||
}
|
||||
if (anchorDuplicate(f, s, threshold)) {
|
||||
mergeInto(s, f);
|
||||
anchor_dups += 1;
|
||||
merged = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!merged) survivors.push(f);
|
||||
}
|
||||
|
|
@ -158,6 +222,7 @@ export function dedupFindings(sources, opts = {}) {
|
|||
total_out: survivors.length,
|
||||
exact_id_dups,
|
||||
jaccard_dups,
|
||||
anchor_dups,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,14 @@
|
|||
// criteria runner's allowlist, and each `End-to-end:` command also stands
|
||||
// under `## Verification`, where Phase 7 runs it.
|
||||
// Strict mode reports these as errors; --soft downgrades them to warnings.
|
||||
//
|
||||
// Every plan version (the fields are new; a plan without them is unchanged):
|
||||
// - K10: the optional step field `Regression:` holds ONE command the
|
||||
// criteria runner's allowlist runs (first backticked span, no pipe, no
|
||||
// `&&`); the executor runs it green before the step's checkpoint. One that
|
||||
// can never run is an error (a warning under --soft).
|
||||
// - K11: `On failure:` opens with revert | retry | skip | escalate, the words
|
||||
// /trekexecute reads; any other opening is a warning.
|
||||
|
||||
import { readFileSync, existsSync, realpathSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
|
@ -29,6 +37,7 @@ import {
|
|||
sliceSteps, validatePlanHeadings, extractPlanVersion, extractPlanCeremony,
|
||||
parseStepFields, findSection, splitSections, findModuleCycle, parseMermaid, fencedBlocks,
|
||||
versionAtLeast, PLAN_VERSION_CURRENT, PLAN_VERSIONS_SUPPORTED, CEREMONY_VALUES,
|
||||
ON_FAILURE_ACTIONS, onFailureAction,
|
||||
} from '../parsers/plan-schema.mjs';
|
||||
import { validateAllManifests } from '../parsers/manifest-yaml.mjs';
|
||||
import { parseDocument } from '../util/frontmatter.mjs';
|
||||
|
|
@ -125,6 +134,36 @@ function checkRunnable(text, sections, opts) {
|
|||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* K10 and K11. `Verify:` is both the step's one test command and its proof of
|
||||
* red, so a step whose red test is a shell script could not also run the frozen
|
||||
* suite before its checkpoint; the plan wrote "run npm test before the
|
||||
* checkpoint" in prose that nothing enforced (voyage-lekerepo trial 2,
|
||||
* 2026-10-08). `Regression:` is that command, checked like `Verify:`.
|
||||
*/
|
||||
function checkRecovery(sections, opts) {
|
||||
const errors = [];
|
||||
const warnings = [];
|
||||
for (const s of sections) {
|
||||
const f = parseStepFields(s.body);
|
||||
if (f.has('Regression')) {
|
||||
const r = runnableCheck(firstCommand(f.get('Regression').text), opts);
|
||||
if (!r.runnable) {
|
||||
errors.push(issue('PLAN_STEP_REGRESSION_NOT_RUNNABLE',
|
||||
`Step ${s.n}: \`Regression:\` is not one runnable command (${r.reason})`,
|
||||
'Backtick ONE command the criteria runner allows, unpiped and without `&&` (`npm test`); ' +
|
||||
`the executor runs it green before the checkpoint. ${WAY_IN}`));
|
||||
}
|
||||
}
|
||||
if (f.has('On failure') && onFailureAction(f.get('On failure').text) === null) {
|
||||
warnings.push(issue('PLAN_STEP_ON_FAILURE_UNKNOWN',
|
||||
`Step ${s.n}: \`On failure:\` does not open with a recovery word the executor reads`,
|
||||
`Open with one of ${ON_FAILURE_ACTIONS.join(' | ')}, then the specifics (\`retry once, then escalate\`).`));
|
||||
}
|
||||
}
|
||||
return { errors, warnings };
|
||||
}
|
||||
|
||||
/** Table body rows (skips the header and the |---| separator). */
|
||||
function tableRows(text) {
|
||||
const rows = text.split('\n').filter(l => l.trim().startsWith('|'));
|
||||
|
|
@ -230,6 +269,11 @@ export function validatePlanContent(rawText, opts = {}) {
|
|||
warnings.push(issue('PLAN_VERSION_MISMATCH', `plan_version=${planVersion}, current target is ${PLAN_VERSION_CURRENT}`));
|
||||
}
|
||||
|
||||
const recovery = checkRecovery(sections, { cwd: opts.cwd });
|
||||
if (strict) errors.push(...recovery.errors);
|
||||
else warnings.push(...recovery.errors);
|
||||
warnings.push(...recovery.warnings);
|
||||
|
||||
const ceremony = extractPlanCeremony(text);
|
||||
if (versionAtLeast(planVersion, '1.8')) {
|
||||
const v18 = [...checkSteps(sections), ...checkRunnable(text, sections, { cwd: opts.cwd })];
|
||||
|
|
|
|||
|
|
@ -610,7 +610,10 @@ export function formatCriteriaEvidence(rep) {
|
|||
for (const r of rep.results) {
|
||||
const cmd = r.command ? `\`${cell(r.command)}\`` : '(none)';
|
||||
const exit = r.exitCode === null ? '—' : `exit ${r.exitCode}`;
|
||||
const evidence = r.status === 'passed' ? '—' : cell(firstLine(r.output)) || '—';
|
||||
// The first line of output on every row, a PASS too: /trekreview tells
|
||||
// the reviewer it gets "the command, exit code and first line of output",
|
||||
// and a PASS row said "—" (voyage-lekerepo trial 2, 2026-10-08).
|
||||
const evidence = cell(firstLine(r.output)) || '—';
|
||||
out.push(`| ${r.label} | ${cmd} | ${MARK[r.status]} | ${exit} | ${evidence} |`);
|
||||
}
|
||||
const s = rep.summary;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue