fix(review): the coordinator gets the brief's choices and writes the brief-conflict mark
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
8cc6ba0181
commit
11315d04eb
4 changed files with 239 additions and 6 deletions
|
|
@ -306,3 +306,154 @@ test('verdict gate — imports illegalBriefLowering from coordinator-contract.mj
|
|||
assert.doesNotMatch(src, /function\s+illegalBriefLowering\b/,
|
||||
'verdict-gate.mjs must not carry its own copy of the rule');
|
||||
});
|
||||
|
||||
// ---- the prose states the same rule (Step 10) --------------------------------
|
||||
//
|
||||
// The coordinator is a model reading agents/review-coordinator.md; the gate
|
||||
// above enforces the rule on what it writes. These pins keep the prose saying
|
||||
// the same rule the code enforces. Every pin reads a bounded cut, never the
|
||||
// whole file, so a phrase elsewhere in the file cannot satisfy it.
|
||||
|
||||
const readDoc = (...rel) => readFileSync(join(ROOT, ...rel), 'utf-8');
|
||||
|
||||
/** The text from `startMarker` up to (not including) the first `endMarker` after it. */
|
||||
function cut(text, startMarker, endMarker, label) {
|
||||
const start = text.indexOf(startMarker);
|
||||
assert.ok(start >= 0, `${label}: start marker missing: ${JSON.stringify(startMarker)}`);
|
||||
const end = text.indexOf(endMarker, start + startMarker.length);
|
||||
assert.ok(end > start, `${label}: end marker ${JSON.stringify(endMarker)} missing after the start`);
|
||||
return text.slice(start, end);
|
||||
}
|
||||
|
||||
/** The sentence from `startPhrase` to its full stop (the first '.' after it). */
|
||||
function sentenceFrom(text, startPhrase, label) {
|
||||
const start = text.indexOf(startPhrase);
|
||||
assert.ok(start >= 0, `${label}: sentence start missing: ${JSON.stringify(startPhrase)}`);
|
||||
const stop = text.indexOf('.', start + startPhrase.length);
|
||||
assert.ok(stop > start, `${label}: the sentence has no full stop`);
|
||||
return text.slice(start, stop + 1);
|
||||
}
|
||||
|
||||
/** One hard-rule bullet, from its bold lead to the next bullet. */
|
||||
function hardRule(coord, lead) {
|
||||
const at = coord.indexOf('\n## Hard rules');
|
||||
assert.ok(at >= 0, 'the ## Hard rules section exists');
|
||||
const rules = coord.slice(at);
|
||||
return cut(`${rules}\n- **`, `- **${lead}**`, '\n- **', `hard rule "${lead}"`);
|
||||
}
|
||||
|
||||
test('trekreview Phase 6 — passes the brief\'s Non-Goals, Constraints, Preferences and Open Questions decisions to the coordinator', () => {
|
||||
const cmd = readDoc('commands', 'trekreview.md');
|
||||
const p6 = cut(cmd, '## Phase 6 — Coordinator dedup + verdict', '\n## ', 'Phase 6');
|
||||
for (const section of ['`## Non-Goals`', '`## Constraints`', '`## Preferences`', '`## Open Questions / Assumptions`']) {
|
||||
assert.ok(p6.includes(section), `Phase 6 names the brief section ${section}`);
|
||||
}
|
||||
assert.match(p6, /operator decisions in `## Open Questions \/ Assumptions`/, 'only the operator decisions of Open Questions');
|
||||
assert.match(p6, /verbatim/, 'the choices are passed verbatim');
|
||||
assert.match(p6, /brief choices/i, 'Phase 6 names the input the coordinator reads');
|
||||
});
|
||||
|
||||
test('trekreview workflow path — engine, contradicts_brief and brief_lowered_from are carried into review.md', () => {
|
||||
const cmd = readDoc('commands', 'trekreview.md');
|
||||
const wf = cut(cmd, 'Then continue at **Phase 7**', '\n\n', 'workflow paragraph');
|
||||
for (const field of ['`engine`', '`contradicts_brief`', '`brief_lowered_from`']) {
|
||||
assert.ok(wf.includes(field), `the workflow paragraph names ${field}`);
|
||||
}
|
||||
assert.match(wf, /review\.md/, 'the fields are written into review.md');
|
||||
});
|
||||
|
||||
test('review-coordinator Input — names "Brief choices" (the four sections) and the mode line names the exception', () => {
|
||||
const coord = readDoc('agents', 'review-coordinator.md');
|
||||
const input = cut(coord, '## Input', '\n## Your 4-pass process', 'Input');
|
||||
assert.match(input, /\*\*Brief choices\*\*/);
|
||||
const choices = cut(input, '**Brief choices**', '\n- **', 'Brief choices bullet');
|
||||
for (const section of ['`## Non-Goals`', '`## Constraints`', '`## Preferences`', '`## Open Questions / Assumptions`']) {
|
||||
assert.ok(choices.includes(section), `Brief choices names ${section}`);
|
||||
}
|
||||
assert.match(choices, /verbatim/);
|
||||
const mode = cut(input, '- **Mode**', '\n- **', 'mode line');
|
||||
assert.match(mode, /except the brief-conflict rule, which applies in every mode/);
|
||||
});
|
||||
|
||||
test('review-coordinator Pass 3 — only an engine finding is marked, and the one lowering needs severity MAJOR', () => {
|
||||
const coord = readDoc('agents', 'review-coordinator.md');
|
||||
const p3 = cut(coord, '### Pass 3', 'In `quick` mode', 'Pass 3');
|
||||
assert.match(p3, /`contradicts_brief: "<verbatim>"`/, 'the mark carries the Non-Goal or choice verbatim');
|
||||
assert.match(p3, /engine finding \(`engine: code-review` or `security-review`\)/, 'only an engine finding is marked');
|
||||
assert.match(p3, /only an engine finding is ever marked/);
|
||||
assert.match(p3, /only a `\/code-review` finding at the MAJOR tier and at severity MAJOR then drops to MINOR, with `brief_lowered_from: MAJOR`/,
|
||||
'the one lowering, severity MAJOR required');
|
||||
assert.match(p3, /a BLOCKER-tier, a MINOR-tier and any `\/security-review` finding only get the mark/);
|
||||
assert.match(p3, /no engine finding is otherwise written below its catalogue tier/);
|
||||
assert.match(p3, /`engine` is written only on a finding whose kept Pass 1 survivor itself came from an engine payload/);
|
||||
assert.match(p3, /a reviewer-agent finding \(no `engine`\) and a finding whose `rule_key` is not in the catalogue are never marked/);
|
||||
assert.match(p3, /not the rule key `NON_GOAL_VIOLATED`/);
|
||||
assert.ok(p3.includes('the brief-conflict reason (`contradicts_brief`), a field, never a `suppressed_reason`'),
|
||||
'the mark is named as a field, never a suppression reason');
|
||||
assert.match(p3, /verdict gate refuses \(exit 2\) every other marked or lowered shape/);
|
||||
});
|
||||
|
||||
test('review-coordinator — quick mode and high-effort normalization each still apply the brief-conflict rule', () => {
|
||||
const coord = readDoc('agents', 'review-coordinator.md');
|
||||
const pass3 = coord.indexOf('### Pass 3');
|
||||
assert.ok(pass3 >= 0, 'Pass 3 heading');
|
||||
const quick = cut(coord.slice(pass3), 'In `quick` mode, skip this pass entirely.', '\n\n', 'quick-mode paragraph');
|
||||
assert.match(quick, /brief-conflict rule[^.]*still applies/, 'the quick-mode paragraph says the rule still applies');
|
||||
const high = cut(coord, '**High-effort normalization (v5.1.1):**', '\n### ', 'high-effort paragraph');
|
||||
assert.match(high, /brief-conflict rule[^.]*still applies/, 'the high-effort paragraph says the rule still applies');
|
||||
});
|
||||
|
||||
test('review-coordinator Pass 2 — the trim rule judges a lowerable brief conflict as MINOR, inside its own sentence', () => {
|
||||
const coord = readDoc('agents', 'review-coordinator.md');
|
||||
const s = sentenceFrom(coord, 'The trim decision reads the **catalogue** tier', 'trim rule');
|
||||
assert.match(s, /except a lowerable brief conflict, which is judged as MINOR \(decide the mark before trimming\)/);
|
||||
assert.match(s, /so a mislabelled MAJOR is never cut\.$/, 'the sentence still ends on its own guarantee');
|
||||
});
|
||||
|
||||
test('review-coordinator hard rules — the brief re-evaluation, the severity floor and quick mode each name the exception', () => {
|
||||
const coord = readDoc('agents', 'review-coordinator.md');
|
||||
assert.match(hardRule(coord, 'Bounded operations only.'), /brief-conflict rule/,
|
||||
'"You do NOT re-evaluate findings against the brief" names its one exception');
|
||||
assert.match(hardRule(coord, 'Severity floor is the catalogue.'), /except a lowerable brief conflict, which is MINOR/);
|
||||
assert.match(hardRule(coord, 'Quick mode is documented.'), /except the brief-conflict rule, which applies in every mode/);
|
||||
});
|
||||
|
||||
test('review-coordinator output — the field list and the trailing JSON example carry engine, contradicts_brief, brief_lowered_from', () => {
|
||||
const coord = readDoc('agents', 'review-coordinator.md');
|
||||
const fields = cut(coord, 'Each Findings subsection uses the', '### Trailing JSON block', 'field list');
|
||||
assert.ok(fields.includes('- `- engine: {code-review | security-review, when the finding came from an engine}`'));
|
||||
assert.ok(fields.includes('- `- contradicts_brief: {verbatim Non-Goal or choice, only when set}`'));
|
||||
assert.ok(fields.includes('- `- brief_lowered_from: {MAJOR, only when lowered for the brief}`'));
|
||||
assert.ok(!fields.includes('original_severity'), 'the brief rule does not read original_severity');
|
||||
|
||||
const block = coord.match(/### Trailing JSON block\n\nThe LAST fenced block in the file is a `json` block:\n\n```json\n([\s\S]*?)\n```/);
|
||||
assert.ok(block, 'the trailing JSON example is present');
|
||||
const objects = block[1].match(/\{[^{}]*\}/g) ?? [];
|
||||
const lowered = objects.find((o) => /"brief_lowered_from":\s*"MAJOR"/.test(o));
|
||||
assert.ok(lowered, 'the JSON example shows a finding with brief_lowered_from');
|
||||
assert.match(lowered, /"engine":\s*"code-review"/);
|
||||
assert.match(lowered, /"contradicts_brief":\s*"[^"]+"/);
|
||||
assert.match(lowered, /"severity":\s*"MINOR"/);
|
||||
assert.match(lowered, /"rule_key":\s*"CODE_REVIEW_DEFECT"/);
|
||||
});
|
||||
|
||||
test('Handover 6 — names the fields, engine findings only, the one lowering and the five shapes the gate refuses', () => {
|
||||
const contracts = readDoc('docs', 'HANDOVER-CONTRACTS.md');
|
||||
const h6 = cut(contracts, '## Handover 6 — `review.md` → plan', '\n## Handover 7', 'Handover 6');
|
||||
for (const field of ['`engine`', '`contradicts_brief`', '`brief_lowered_from`']) {
|
||||
assert.ok(h6.includes(field), `Handover 6 names ${field}`);
|
||||
}
|
||||
assert.match(h6, /only an engine finding is (?:ever )?marked/i, 'engine findings only');
|
||||
assert.match(h6, /`\/code-review` MAJOR-tier finding[^.]*MINOR[^.]*`brief_lowered_from: MAJOR`/, 'the one legal lowering');
|
||||
assert.match(h6, /exits 2/);
|
||||
const shapes = [
|
||||
/any other `brief_lowered_from`/,
|
||||
/a mark on a reviewer-agent finding/,
|
||||
/a marked finding below its tier/,
|
||||
/a marked finding with an unknown key/,
|
||||
/an engine finding below its tier, marked or not/,
|
||||
];
|
||||
for (const shape of shapes) assert.match(h6, shape, `Handover 6 names the refused shape ${shape}`);
|
||||
assert.match(h6, /cannot check an `engine` label on the prose path/);
|
||||
assert.match(h6, /skips a lowered finding/, 'a plan made from the review skips a lowered (MINOR) finding');
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue