fix(gate): part 2 verifier findings — non-object findings, carried backed copy, rule_keys, marks, approval prose, brief_path, engine put-back once
1. plan-review-dedup skips only a null or undefined finding; any other entry that is not an object (a string, a number, an array) goes on as before E, a finding with no valid severity that the plan gate counts as a blocker (it was dropped, so a string blocker recorded PASS). 2. plan-gate --record: a previous finding that rule A carried as counted and whose id is backed this round leaves the counted set, the counts, the verdict and the stop rule's summary (it was counted and listed in known_outside at once: BLOCK, not-fewer, and --accept-open refused). 3. A known_outside entry keeps rule_keys; extractVerdict accepts it (a list of strings), and the bare gate re-checks with the same keys the recorder used, so a merged finding whose security key is only in rule_keys is a WARN, not a STOP. 4. plan-critic, scope-guardian and /trekplan Phase 9's round-2 sentences say the Critic Verdict also lists known_outside and each is marked by id in previous. Code already accepts such a mark (dedup passes it; countRound ignores ids it does not carry; --record removes the entry on true); pinned in diff mode and full count. 5. The approval prose names ## Threat Model where it named only ## Intent + ## Goal: skills/annotating/SKILL.md (description, step 4, guard list), docs/command-modes.md, Handover 1's intent_approved_hash row, README (approval, Approve row, intent approval gate) and CLAUDE.md's /trekbrief row. 6. threat-model.mjs gains usableInRepo (a brief path relative to the repo top level) and repoRelativeBrief; --record stores brief_path relative to the top level, never absolute, and both gates read it from there, so the gate run from a subdirectory reads the same brief. reviewThreatModel uses usableInRepo. Handover 4 and the gate header say so. 7. review-finalize's outside pass puts an entry back only when no finding carries its row yet (engine-compare rowOf): an engine finding listed in known_outside while its payload is saved went back twice. 8. A test runs scripts/release-check.mjs on a review whose known_outside entry the brief does not back: exit 2 (control: backed passes the review check). Existing tests changed: plan-gate-threat-model's entry-keys assertion now expects rule_keys (item 3), and its planted non-security entry drops the security key from rule_keys too, since the gate reads rule_keys (item 3). Part 2 fix round of the E plan. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
c47061191c
commit
1be961aab1
17 changed files with 298 additions and 52 deletions
|
|
@ -36,8 +36,10 @@
|
|||
// `threat_model_ref` (the text of the brief's `Outside:` line it quotes), and
|
||||
// a merge keeps it only when every merged copy carries the same one; one copy
|
||||
// without it, or two different ones, drop it, so the plan gate counts the
|
||||
// merged finding in full. A non-object in a reviewer's findings list (a
|
||||
// `null`) is skipped instead of taking the CLI down (the S160 known issue).
|
||||
// merged finding in full. A `null` or `undefined` in a reviewer's findings
|
||||
// list is skipped instead of taking the CLI down (the S160 known issue); any
|
||||
// other entry that is not an object (a string, a number, an array) stays a
|
||||
// finding with no valid severity, which the plan gate counts as a blocker.
|
||||
//
|
||||
// Provenance is preserved on the surviving finding's `raised_by` array.
|
||||
// Severity is kept (normalized to blocker | major | minor, else null). A merge
|
||||
|
|
@ -218,8 +220,9 @@ function normalizeAgentPayload(payload, fallbackAgent) {
|
|||
const agent = (typeof payload.agent === 'string' && payload.agent.length > 0)
|
||||
? payload.agent
|
||||
: fallbackAgent;
|
||||
// A non-object entry (a `null`) is no finding: skipped, never thrown on (S160).
|
||||
const findings = Array.isArray(payload.findings) ? payload.findings.filter(f => f !== null && typeof f === 'object') : [];
|
||||
// A `null` or `undefined` entry holds nothing: skipped, never thrown on (S160).
|
||||
// Any other entry, an object or not, goes on as before (no valid severity = a blocker).
|
||||
const findings = Array.isArray(payload.findings) ? payload.findings.filter(f => f !== null && f !== undefined) : [];
|
||||
const previous = Array.isArray(payload.previous) ? payload.previous : [];
|
||||
return { agent, pairs: findings.map(f => ({ agent, finding: f })), previous };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,8 +27,11 @@
|
|||
// goes back into the findings at its catalogue tier with no id, a `### `
|
||||
// entry with a `**Source:**` line saying why, the frontmatter list, counts,
|
||||
// a verdict that is only raised and, as a BLOCKER, its `### Step R<n>`.
|
||||
// The entries that stay are listed under `## Outside the Threat Model`;
|
||||
// a malformed entry stays too (the verdict gate exits 2 on it).
|
||||
// An entry whose row a finding already carries (the engine pass put the
|
||||
// same engine finding back, engine-compare.mjs `rowOf`) leaves the list
|
||||
// without a second copy. The entries that stay are listed under
|
||||
// `## Outside the Threat Model`; a malformed entry stays too (the verdict
|
||||
// gate exits 2 on it).
|
||||
// - instruction, right after engine: a finding under `CLAUDE_MD_DEVIATION`
|
||||
// or `CLAUDE_MD_STALE` whose `instruction_ref` is missing or names a file
|
||||
// outside the repo's tracked instruction set (lib/review/instruction-scope.mjs
|
||||
|
|
@ -122,7 +125,7 @@ import { isSecurityFinding } from './review-answers.mjs';
|
|||
import { carry, frontmatterField, MAX_REVIEW_ROUNDS, runId } from './review-rounds.mjs';
|
||||
import { applyPolicy, readPolicy } from './review-policy.mjs';
|
||||
import { ENGINES } from './engine-findings.mjs';
|
||||
import { ENGINE_OF_REVIEWER, putBack } from './engine-compare.mjs';
|
||||
import { ENGINE_OF_REVIEWER, putBack, rowOf, trimmedTitle } from './engine-compare.mjs';
|
||||
import { backedLine, isKnownEntry, reviewThreatModel } from './threat-model.mjs';
|
||||
import { getRule } from './rule-catalogue.mjs';
|
||||
import { isMain } from '../util/is-main.mjs';
|
||||
|
|
@ -288,9 +291,15 @@ function passOutside(payload, ctx) {
|
|||
else kept.push(e);
|
||||
}
|
||||
const why = tm ? 'no `Outside:` line of the brief equals its threat_model_ref, or it is not a Voyage security finding' : use.why;
|
||||
// A row a finding already carries (the engine pass put the same engine
|
||||
// finding back) is counted once: the entry leaves the list, no second copy.
|
||||
const rows = new Set(payload.findings.filter(isPlainObject).map(rowOf));
|
||||
for (const e of back) {
|
||||
const rule = getRule(e.rule_key.trim().toUpperCase());
|
||||
const f = { ...e, id: null, rule_key: rule ? rule.rule_key : e.rule_key, severity: rule ? rule.severity : 'BLOCKER' };
|
||||
const held = rows.has(rowOf(f)) || (typeof f.title === 'string' && rows.has(rowOf({ ...f, title: trimmedTitle(f.title) })));
|
||||
if (held) continue;
|
||||
rows.add(rowOf(f));
|
||||
payload.findings.push(f);
|
||||
ctx.added.add(f);
|
||||
ctx.outsideBack.set(f, why);
|
||||
|
|
|
|||
|
|
@ -21,9 +21,16 @@
|
|||
// Otherwise every security finding counts, and `why` says why.
|
||||
// backedLine(finding, tm) -> the `Outside:` text the finding's
|
||||
// `threat_model_ref` equals exactly, else null.
|
||||
// reviewThreatModel(reviewText, repo) -> usable() for the brief a review
|
||||
// names in its `brief_path` (a relative path is read from the repo's top
|
||||
// level); the review side's `known_outside` (E 2.5) is checked with it.
|
||||
// usableInRepo(briefPath, repo) -> usable() for a brief path relative to
|
||||
// the repo's top level (an absolute one as it is); both gates read a
|
||||
// stored `brief_path` this way, whatever directory they run in.
|
||||
// repoRelativeBrief(briefPath, repo) -> a brief path given from the
|
||||
// working directory, made relative to the repo's top level (forward
|
||||
// slashes; `..` when outside it), never absolute: what the plan gate
|
||||
// stores as `brief_path`.
|
||||
// reviewThreatModel(reviewText, repo) -> usableInRepo() for the brief a
|
||||
// review names in its `brief_path`; the review side's `known_outside`
|
||||
// (E 2.5) is checked with it.
|
||||
// isKnownEntry(e) -> a review `known_outside` entry's shape: a
|
||||
// finding with a string `rule_key` and `file`, an integer `line` >= 0
|
||||
// and a non-empty `threat_model_ref`.
|
||||
|
|
@ -33,7 +40,7 @@
|
|||
// every run, so a finding whose line is gone counts in full.
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { isAbsolute, join } from 'node:path';
|
||||
import { isAbsolute, join, relative, sep } from 'node:path';
|
||||
import { parseDocument } from '../util/frontmatter.mjs';
|
||||
import { verifyLog, normalizeHex, normalizeSubject, resolveRepo } from '../util/approval-log.mjs';
|
||||
import { checkIntentApprovalContent } from '../validators/intent-approval.mjs';
|
||||
|
|
@ -93,8 +100,38 @@ export function usable(briefPath, repo) {
|
|||
}
|
||||
|
||||
/**
|
||||
* The threat model a review applies: usable() for the brief at the review's
|
||||
* frontmatter `brief_path`, a relative path read from the repo's top level.
|
||||
* usable() for a brief path relative to the repo's top level (an absolute
|
||||
* path as it is), so a stored `brief_path` reads the same brief from any
|
||||
* working directory.
|
||||
* @param {string} briefPath
|
||||
* @param {string} [repo] - the repo (default: the git top level of the working directory)
|
||||
* @returns {{usable: boolean, why: string|null, threatModel: object}}
|
||||
*/
|
||||
export function usableInRepo(briefPath, repo) {
|
||||
if (typeof briefPath !== 'string' || briefPath.trim() === '') return { usable: false, why: 'no brief path was given', threatModel: EMPTY };
|
||||
let root;
|
||||
try { root = resolveRepo(repo); } catch (e) {
|
||||
return { usable: false, why: `the repo could not be read: ${e.message}`, threatModel: EMPTY };
|
||||
}
|
||||
return usable(isAbsolute(briefPath) ? briefPath : join(root, briefPath), root);
|
||||
}
|
||||
|
||||
/**
|
||||
* A brief path given from the working directory, relative to the repo's top
|
||||
* level: forward slashes, `..` when the brief is outside the repo, never absolute.
|
||||
* @param {string} briefPath
|
||||
* @param {string} [repo] - the repo (default: the git top level of the working directory)
|
||||
* @returns {string}
|
||||
*/
|
||||
export function repoRelativeBrief(briefPath, repo) {
|
||||
const root = resolveRepo(repo);
|
||||
const s = normalizeSubject(root, briefPath);
|
||||
return isAbsolute(s) ? relative(root, s).split(sep).join('/') : s;
|
||||
}
|
||||
|
||||
/**
|
||||
* The threat model a review applies: usableInRepo() for the brief at the
|
||||
* review's frontmatter `brief_path`, a relative path read from the repo's top level.
|
||||
* @param {string} reviewText - review.md
|
||||
* @param {string} [repo] - the repo (default: the git top level of the working directory)
|
||||
* @returns {{usable: boolean, why: string|null, threatModel: object}}
|
||||
|
|
@ -103,11 +140,7 @@ export function reviewThreatModel(reviewText, repo) {
|
|||
const doc = parseDocument(typeof reviewText === 'string' ? reviewText : '');
|
||||
const bp = doc.parsed?.frontmatter?.brief_path;
|
||||
if (typeof bp !== 'string' || bp.trim() === '') return { usable: false, why: 'the review names no brief_path', threatModel: EMPTY };
|
||||
let root;
|
||||
try { root = resolveRepo(repo); } catch (e) {
|
||||
return { usable: false, why: `the repo could not be read: ${e.message}`, threatModel: EMPTY };
|
||||
}
|
||||
return usable(isAbsolute(bp) ? bp : join(root, bp), root);
|
||||
return usableInRepo(bp, repo);
|
||||
}
|
||||
|
||||
/** A review `known_outside` entry's shape (E 2.5): string rule_key and file, integer line >= 0, non-empty threat_model_ref. */
|
||||
|
|
|
|||
|
|
@ -122,9 +122,12 @@
|
|||
// log). A backed finding — Voyage's own security finding (no engine) whose
|
||||
// `threat_model_ref` equals an `Outside:` line's text exactly — leaves the
|
||||
// counted set and the stop rule and goes to the optional verdict field
|
||||
// `known_outside` ({id, rule_key, text, line}), carried to later rounds until
|
||||
// a reviewer marks it resolved (or it counts again under its id); the verdict
|
||||
// stores `brief_path` (carried when a later round has no --brief). A threat
|
||||
// `known_outside` ({id, rule_key, rule_keys, text, line}), carried to later
|
||||
// rounds until a reviewer marks it resolved (or it counts again under its id);
|
||||
// a copy of it that rule A carried from an earlier round as counted leaves the
|
||||
// count too. The verdict stores `brief_path`, relative to the repo's top level
|
||||
// (carried when a later round has no --brief), and the gate reads it from
|
||||
// there whatever its working directory. A threat
|
||||
// model that is not usable backs nothing: every security finding counts, and
|
||||
// `threat_model.why` says why. The bare gate re-reads the brief at
|
||||
// `brief_path` on every run: an entry still backed is a warning
|
||||
|
|
@ -140,10 +143,10 @@ import { createHash } from 'node:crypto';
|
|||
import { dirname, join, resolve } from 'node:path';
|
||||
import { validatePlanContent } from './plan-validator.mjs';
|
||||
import { splitSections, fenceTracker, extractPlanVersion, versionAtLeast } from '../parsers/plan-schema.mjs';
|
||||
import { normalizeSeverity, isSecurity } from '../review/plan-review-dedup.mjs';
|
||||
import { normalizeSeverity, isSecurity, SEVERITIES } from '../review/plan-review-dedup.mjs';
|
||||
import { countRound, lineDiff, oneLine } from '../review/critic-diff.mjs';
|
||||
import { roundSummary, decideStop, describeStop, STOP_REASONS } from '../review/stop-rule.mjs';
|
||||
import { usable as threatModelUsable, backedLine } from '../review/threat-model.mjs';
|
||||
import { usableInRepo, repoRelativeBrief, backedLine } from '../review/threat-model.mjs';
|
||||
import { resolveRepo } from '../verification/repo-tools.mjs';
|
||||
import { isMain } from '../util/is-main.mjs';
|
||||
|
||||
|
|
@ -281,13 +284,21 @@ function okStopRule(s) {
|
|||
|
||||
const isStrOrNull = x => x === null || typeof x === 'string';
|
||||
|
||||
/** Optional `known_outside`: a list of {id: string|null, rule_key: string|null, text: string, line: non-empty string}. */
|
||||
/** Optional `known_outside`: a list of {id: string|null, rule_key: string|null, rule_keys?: string[], text: string, line: non-empty string}. */
|
||||
function okKnownOutside(k) {
|
||||
if (k === undefined) return true;
|
||||
return Array.isArray(k) && k.every(e => isPlainObject(e) && isStrOrNull(e.id) && isStrOrNull(e.rule_key) &&
|
||||
(e.rule_keys === undefined || (Array.isArray(e.rule_keys) && e.rule_keys.every(x => typeof x === 'string'))) &&
|
||||
typeof e.text === 'string' && typeof e.line === 'string' && e.line !== '');
|
||||
}
|
||||
|
||||
/** A counted finding a reviewer rated higher than it counts (rule A or B). */
|
||||
const isLowered = x => {
|
||||
const o = normalizeSeverity(x.original_severity);
|
||||
const s = normalizeSeverity(x.severity);
|
||||
return o !== null && s !== null && SEVERITIES.indexOf(o) < SEVERITIES.indexOf(s);
|
||||
};
|
||||
|
||||
/** Optional `brief_path`: a non-empty string. */
|
||||
const okBriefPath = p => p === undefined || (typeof p === 'string' && p !== '');
|
||||
|
||||
|
|
@ -407,9 +418,15 @@ export function recordVerdict(text, dedup, opts = {}) {
|
|||
const counted = [];
|
||||
for (const f of dedup.findings) {
|
||||
const line = tm ? backedLine(f, tm) : null;
|
||||
if (line !== null) backed.push({ id: f.id ?? null, rule_key: f.rule_key ?? null, text: typeof f.text === 'string' ? f.text : '', line });
|
||||
else counted.push(f);
|
||||
if (line === null) { counted.push(f); continue; }
|
||||
// rule_keys kept, so the bare gate re-checks the entry with the same keys (keysOf).
|
||||
backed.push({
|
||||
id: f.id ?? null, rule_key: f.rule_key ?? null,
|
||||
...(Array.isArray(f.rule_keys) ? { rule_keys: f.rule_keys.filter(k => typeof k === 'string') } : {}),
|
||||
text: typeof f.text === 'string' ? f.text : '', line,
|
||||
});
|
||||
}
|
||||
const backedIds = new Set(backed.map(e => e.id).filter(id => id !== null));
|
||||
const c = countRound({
|
||||
round,
|
||||
previous: prev.verdict ?? null,
|
||||
|
|
@ -420,7 +437,12 @@ export function recordVerdict(text, dedup, opts = {}) {
|
|||
findings: counted,
|
||||
marks: Array.isArray(dedup.marks) ? dedup.marks : [],
|
||||
});
|
||||
const findings = c.findings;
|
||||
// A previous finding countRound carried (rule A) whose id is backed this
|
||||
// round leaves the counted set too: listed once, in known_outside, and out
|
||||
// of the counts, the verdict and the stop rule's summary.
|
||||
const leaving = c.findings.filter(x => x.carried_from !== undefined && x.carried_from !== null && backedIds.has(x.id));
|
||||
const findings = c.findings.filter(x => !leaving.includes(x));
|
||||
const lowered = c.lowered - leaving.filter(isLowered).length;
|
||||
const counts = { blocker: 0, major: 0, minor: 0 };
|
||||
for (const f of findings) counts[f.severity] += 1;
|
||||
const verdict = {
|
||||
|
|
@ -441,7 +463,6 @@ export function recordVerdict(text, dedup, opts = {}) {
|
|||
// entries that no reviewer marked resolved and that are not reported again
|
||||
// under their id (backed again: listed once; counted: in full, not listed).
|
||||
const marks = Array.isArray(dedup.marks) ? dedup.marks : [];
|
||||
const backedIds = new Set(backed.map(e => e.id).filter(id => id !== null));
|
||||
const countedIds = new Set(findings.map(x => x.id).filter(id => typeof id === 'string'));
|
||||
const kept = (Array.isArray(prev.verdict?.known_outside) ? prev.verdict.known_outside : [])
|
||||
.filter(e => !knownResolved(e, marks) && !backedIds.has(e.id) && !countedIds.has(e.id));
|
||||
|
|
@ -459,7 +480,7 @@ export function recordVerdict(text, dedup, opts = {}) {
|
|||
? { applied: false, why: 'no --brief given' }
|
||||
: { applied: use.usable === true, why: use.usable === true ? null : (use.why ?? 'the threat model is not usable') };
|
||||
return {
|
||||
code: 0, text: placeSection(text, renderSection(verdict)), verdict, lowered: c.lowered, threat_model: threat,
|
||||
code: 0, text: placeSection(text, renderSection(verdict)), verdict, lowered, threat_model: threat,
|
||||
message: `round ${round} recorded: ${verdict.verdict}` +
|
||||
(knownOutside.length > 0 ? `; ${knownOutside.length} finding(s) outside the threat model listed as known issues` : '') +
|
||||
(use !== undefined && !threat.applied ? `; threat model not applied, every security finding counts: ${threat.why}` : '') +
|
||||
|
|
@ -548,7 +569,8 @@ function checkKnownOutside(entries, use, reasons, warnings) {
|
|||
const backed = [];
|
||||
const unbacked = [];
|
||||
for (const e of entries) {
|
||||
(tm && backedLine({ rule_key: e.rule_key, threat_model_ref: e.line }, tm) !== null ? backed : unbacked).push(e);
|
||||
const asFinding = { rule_key: e.rule_key, ...(Array.isArray(e.rule_keys) ? { rule_keys: e.rule_keys } : {}), threat_model_ref: e.line };
|
||||
(tm && backedLine(asFinding, tm) !== null ? backed : unbacked).push(e);
|
||||
}
|
||||
if (backed.length > 0) {
|
||||
warnings.push({
|
||||
|
|
@ -631,8 +653,10 @@ export function gatePlan(rawText, opts = {}) {
|
|||
}
|
||||
const known = Array.isArray(v.known_outside) ? v.known_outside : [];
|
||||
if (known.length > 0) {
|
||||
// brief_path is relative to the repo's top level, so the gate reads the
|
||||
// same brief from any working directory inside the repo.
|
||||
const use = opts.threatModel ?? (typeof v.brief_path === 'string'
|
||||
? threatModelUsable(v.brief_path)
|
||||
? usableInRepo(v.brief_path)
|
||||
: { usable: false, why: 'the verdict names no brief_path', threatModel: null });
|
||||
checkKnownOutside(known, use, reasons, warnings);
|
||||
}
|
||||
|
|
@ -826,8 +850,9 @@ if (isMain(import.meta.url)) {
|
|||
const dir = snapshotDir(planPath);
|
||||
const opts = { extraRound: args.includes('--extra-round'), reviewed, planPaths: planPathsOf(planPath) };
|
||||
if (briefArg !== undefined) {
|
||||
opts.briefPath = briefArg;
|
||||
opts.threatModel = threatModelUsable(briefArg);
|
||||
// Stored relative to the repo's top level, never absolute; read from there.
|
||||
opts.briefPath = repoRelativeBrief(briefArg);
|
||||
opts.threatModel = usableInRepo(opts.briefPath);
|
||||
}
|
||||
const lb = loadBase(text, dir);
|
||||
if (lb.verdict) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue