757 lines
33 KiB
JavaScript
757 lines
33 KiB
JavaScript
// tests/lib/review-answers.test.mjs
|
|
// [RAP] fix round 4.5b2 part B, Step 3 (R6): a commit that only appends review
|
|
// answers (K2 `review` records) to the top-level approval log after the
|
|
// reviewed commit keeps the review current. Any other change to the log stays
|
|
// `review-stale:`. Tests run the verdict gate's CLI only, in throwaway git
|
|
// repos with no hooks; every record a tagged test needs is written with
|
|
// `lib/util/approval-log.mjs append`, never by hand.
|
|
// `keep:` cases hold at the base already (today's R6 rejects any change to the
|
|
// log) and stay untagged.
|
|
|
|
import { after, test } from 'node:test';
|
|
import { strict as assert } from 'node:assert';
|
|
import { spawnSync } from 'node:child_process';
|
|
import { createHash } from 'node:crypto';
|
|
import {
|
|
appendFileSync, chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync,
|
|
} from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { dirname, join } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { MARK_START, MARK_END, POINTER_START, POINTER_LINE, POINTER_END } from '../../lib/review/recurrence.mjs';
|
|
import { GENESIS, LOG_RELPATH, recordHash } from '../../lib/util/approval-log.mjs';
|
|
import { childEnv } from '../helpers/child-env.mjs';
|
|
|
|
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
|
|
const GATE = join(ROOT, 'lib', 'review', 'verdict-gate.mjs');
|
|
const APPROVAL_LOG = join(ROOT, 'lib', 'util', 'approval-log.mjs');
|
|
|
|
const LESSONS = '.claude/rules/voyage-review-lessons.md';
|
|
const POINTER = `${POINTER_START}\n${POINTER_LINE}\n${POINTER_END}\n`;
|
|
const LESSONS_BEFORE = `# Review lessons\n\n${MARK_START}\nEach line: a finding class.\n${MARK_END}\n\nKept by hand.\n`;
|
|
const LESSON = '- [MISSING_TEST] (check) Mistake: m. · Fix: f. · How to check: c.';
|
|
const LESSONS_AFTER = `# Review lessons\n\n${MARK_START}\nEach line: a finding class.\n${LESSON}\n${MARK_END}\n\nKept by hand.\n`;
|
|
const REVIEW_REL = '.claude/projects/2026-11-02-s/review.md';
|
|
const NEW_REASON = /\.claude\/voyage\/approvals\.jsonl changed other than by appended review answers \(commit [0-9a-f]{8}\)/;
|
|
|
|
const hex = (c) => c.repeat(64);
|
|
|
|
/** A minimal ALLOW review of `sha` (shape copied from tests/lib/verdict-gate.test.mjs `review`). */
|
|
function review(sha) {
|
|
const id = '1'.padStart(40, 'a');
|
|
const json = JSON.stringify({
|
|
verdict: 'ALLOW',
|
|
counts: { BLOCKER: 0, MAJOR: 0, MINOR: 1, SUGGESTION: 0 },
|
|
allow_blocked_by: [],
|
|
findings: [{ id, severity: 'MINOR', rule_key: 'MISSING_TEST', file: 'a.mjs', line: 1, title: 't', detail: 'd', recommended_action: 'Add a test.' }],
|
|
}, null, 2);
|
|
return `---
|
|
type: trekreview
|
|
review_version: "1.1"
|
|
task: "t"
|
|
slug: s
|
|
project_dir: .claude/projects/2026-11-02-s/
|
|
brief_path: .claude/projects/2026-11-02-s/brief.md
|
|
scope_sha_end: ${sha}
|
|
reviewed_files_count: 1
|
|
verdict: ALLOW
|
|
findings:
|
|
- ${id}
|
|
---
|
|
|
|
# Review: t
|
|
|
|
## Executive Summary
|
|
|
|
x
|
|
|
|
## Coverage
|
|
|
|
| File | Treatment | Reason |
|
|
|------|-----------|--------|
|
|
| a.mjs | summary-only | default |
|
|
|
|
## Remediation Summary
|
|
|
|
x
|
|
|
|
\`\`\`json
|
|
${json}
|
|
\`\`\`
|
|
`;
|
|
}
|
|
|
|
function g(cwd, ...args) {
|
|
const r = spawnSync('git', args, { cwd, encoding: 'utf8', env: childEnv() });
|
|
if (r.status !== 0) throw new Error(`git ${args.join(' ')}: ${r.stderr}`);
|
|
return r.stdout.trim();
|
|
}
|
|
|
|
function putIn(dir, rel, text) {
|
|
mkdirSync(dirname(join(dir, rel)), { recursive: true });
|
|
writeFileSync(join(dir, rel), text);
|
|
}
|
|
|
|
const made = [];
|
|
after(() => { for (const b of made) rmSync(b, { recursive: true, force: true }); });
|
|
|
|
/**
|
|
* A throwaway repo with one reviewed commit: a.mjs, CLAUDE.md and the lessons
|
|
* file in the project dir (`sub` puts the project in a subdirectory), and the
|
|
* top-level approval log holding the records `seed` names (kinds, appended
|
|
* with the CLI before the reviewed commit; none = no log).
|
|
*/
|
|
function fixture({ sub = '', seed = ['plan'] } = {}) {
|
|
const base = mkdtempSync(join(tmpdir(), 'review-answers-rap-'));
|
|
made.push(base);
|
|
const repo = join(base, 'repo');
|
|
const dir = sub ? join(repo, sub) : repo;
|
|
mkdirSync(dir, { recursive: true });
|
|
mkdirSync(join(base, 'hooks-none'));
|
|
g(repo, 'init', '-q', '-b', 'main');
|
|
appendFileSync(join(repo, '.git', 'config'), '[user]\n\tname = Test\n\temail = test@example.invalid\n'
|
|
+ `[commit]\n\tgpgsign = false\n[core]\n\thooksPath = "${join(base, 'hooks-none')}"\n`);
|
|
putIn(dir, 'a.mjs', 'export const a = 1;\n');
|
|
putIn(dir, 'CLAUDE.md', '# Project\n');
|
|
putIn(dir, LESSONS, LESSONS_BEFORE);
|
|
const append = (kind, subject, sha) => {
|
|
const r = spawnSync(process.execPath, [APPROVAL_LOG, 'append', '--kind', kind, '--subject', subject, '--sha256', sha, '--repo', repo],
|
|
{ cwd: repo, encoding: 'utf8', env: childEnv() });
|
|
assert.equal(r.status, 0, `approval-log append ${kind}: ${r.stdout}${r.stderr}`);
|
|
};
|
|
seed.forEach((kind, i) => append(kind, `${kind}-${i}.md`, hex(String(i + 1))));
|
|
g(repo, 'add', '-A');
|
|
g(repo, 'commit', '-q', '-m', 'feat: the work');
|
|
const reviewed = g(repo, 'rev-parse', 'HEAD');
|
|
const logFile = join(repo, ...LOG_RELPATH.split('/'));
|
|
const commitAll = (msg) => { g(repo, 'add', '-A'); g(repo, 'commit', '-q', '-m', msg); };
|
|
return {
|
|
repo, dir, reviewed, logFile,
|
|
/** A review record for finding `n`, appended by the CLI. */
|
|
answer: (n) => append('review', `${REVIEW_REL}#${String(n).padStart(40, 'a')}`, hex('c')),
|
|
append,
|
|
readLines: () => readFileSync(logFile, 'utf8').split('\n').filter(Boolean),
|
|
writeLines: (lines) => writeFileSync(logFile, lines.map((l) => `${l}\n`).join('')),
|
|
put: (rel, text) => putIn(dir, rel, text),
|
|
commitAll,
|
|
gate: () => {
|
|
const p = join(base, 'review.md');
|
|
writeFileSync(p, review(reviewed));
|
|
return spawnSync(process.execPath, [GATE, '--json', p, '--repo', dir], { encoding: 'utf8', env: childEnv() });
|
|
},
|
|
};
|
|
}
|
|
|
|
/** A record built by hand (keep: cases only), hashed as the K2 log hashes it. */
|
|
function record(fields, { hash } = {}) {
|
|
const rec = { v: 1, ts: '2026-11-02T10:00:00.000Z', kind: 'review', subject: `${REVIEW_REL}#x`, sha256: hex('d'), expires: null, ...fields };
|
|
rec.hash = hash ?? recordHash(rec);
|
|
return JSON.stringify(rec);
|
|
}
|
|
|
|
function assertStale(r) {
|
|
assert.equal(r.status, 2, r.stdout + r.stderr);
|
|
assert.match(r.stdout, /review-stale/);
|
|
}
|
|
|
|
// ---- tagged ----
|
|
|
|
test('[RAP] a commit that appends review records to the approval log after the reviewed commit is not stale', () => {
|
|
// A log that already holds a plan approval, and a log that does not exist at the reviewed commit.
|
|
for (const seed of [['plan'], []]) {
|
|
const fx = fixture({ seed });
|
|
fx.answer(1);
|
|
fx.answer(2);
|
|
fx.commitAll('docs(voyage): review answers (round 1)');
|
|
// Control: the commit after the review really changes the log.
|
|
assert.deepEqual(g(fx.repo, 'diff', '--name-only', `${fx.reviewed}..HEAD`).split('\n'), [LOG_RELPATH]);
|
|
const r = fx.gate();
|
|
assert.equal(r.status, 0, `seed ${JSON.stringify(seed)}: ${r.stdout}${r.stderr}`);
|
|
// The lessons commit after the answers keeps it current too.
|
|
fx.put(LESSONS, LESSONS_AFTER);
|
|
fx.put('CLAUDE.md', `# Project\n\n${POINTER}`);
|
|
fx.commitAll('docs(voyage): review lessons (round 1)');
|
|
assert.equal(fx.gate().status, 0, `seed ${JSON.stringify(seed)} after lessons`);
|
|
}
|
|
// Any other change to the log is stale, with the reason that names the rule.
|
|
const fx = fixture();
|
|
const [first] = fx.readLines();
|
|
const rewritten = record({ ...JSON.parse(first), hash: undefined, ts: '2026-11-02T11:00:00.000Z' });
|
|
fx.writeLines([rewritten]);
|
|
fx.commitAll('chore: rewrite the log');
|
|
const r = fx.gate();
|
|
assertStale(r);
|
|
assert.match(r.stdout, NEW_REASON);
|
|
});
|
|
|
|
test('[RAP] a project in a subdirectory: a commit appending review records to the top-level log stays current', () => {
|
|
const fx = fixture({ sub: 'plugin' });
|
|
fx.answer(1);
|
|
fx.commitAll('docs(voyage): review answers (round 1)');
|
|
// Control: the log changed is the top-level one, not one under the project prefix.
|
|
assert.deepEqual(g(fx.repo, 'diff', '--name-only', `${fx.reviewed}..HEAD`).split('\n'), [LOG_RELPATH]);
|
|
const r = fx.gate();
|
|
assert.equal(r.status, 0, r.stdout + r.stderr);
|
|
});
|
|
|
|
// ---- keep: (hold at the base) ----
|
|
|
|
test('keep: a commit that rewrites an earlier record (re-hashed) and appends a review record is stale', () => {
|
|
const fx = fixture();
|
|
const [first] = fx.readLines();
|
|
const old = JSON.parse(first);
|
|
delete old.hash;
|
|
const rewritten = record({ ...old, sha256: hex('e') });
|
|
const next = record({ seq: 2, prev: JSON.parse(rewritten).hash });
|
|
fx.writeLines([rewritten, next]);
|
|
fx.commitAll('docs(voyage): review answers (round 1)');
|
|
assertStale(fx.gate());
|
|
});
|
|
|
|
test('keep: a commit that removes a record and appends a review record is stale', () => {
|
|
const fx = fixture();
|
|
fx.writeLines([record({ seq: 1, prev: GENESIS })]);
|
|
fx.commitAll('docs(voyage): review answers (round 1)');
|
|
assertStale(fx.gate());
|
|
});
|
|
|
|
test('keep: a commit that reorders the records (re-chained) and appends a review record is stale', () => {
|
|
const fx = fixture({ seed: ['intent', 'plan'] });
|
|
const [a, b] = fx.readLines().map((l) => { const r = JSON.parse(l); delete r.hash; return r; });
|
|
const one = record({ ...b, seq: 1, prev: GENESIS });
|
|
const two = record({ ...a, seq: 2, prev: JSON.parse(one).hash });
|
|
const three = record({ seq: 3, prev: JSON.parse(two).hash });
|
|
fx.writeLines([one, two, three]);
|
|
fx.commitAll('docs(voyage): review answers (round 1)');
|
|
assertStale(fx.gate());
|
|
});
|
|
|
|
test('keep: a commit that appends a plan record is stale', () => {
|
|
const fx = fixture();
|
|
fx.append('plan', 'plan-new.md', hex('f'));
|
|
fx.commitAll('docs(voyage): plan approval');
|
|
assertStale(fx.gate());
|
|
});
|
|
|
|
test('keep: a commit that appends a merge record is stale', () => {
|
|
const fx = fixture();
|
|
fx.answer(1);
|
|
fx.append('merge', 'main', hex('f'));
|
|
fx.commitAll('docs(voyage): review answers (round 1)');
|
|
assertStale(fx.gate());
|
|
});
|
|
|
|
test('keep: a review record whose prev does not link is stale', () => {
|
|
const fx = fixture();
|
|
fx.writeLines([...fx.readLines(), record({ seq: 2, prev: GENESIS })]);
|
|
fx.commitAll('docs(voyage): review answers (round 1)');
|
|
assertStale(fx.gate());
|
|
});
|
|
|
|
test('keep: a review record whose seq skips is stale', () => {
|
|
const fx = fixture();
|
|
const lines = fx.readLines();
|
|
fx.writeLines([...lines, record({ seq: 3, prev: JSON.parse(lines[0]).hash })]);
|
|
fx.commitAll('docs(voyage): review answers (round 1)');
|
|
assertStale(fx.gate());
|
|
});
|
|
|
|
test('keep: a review record whose hash does not match is stale', () => {
|
|
const fx = fixture();
|
|
const lines = fx.readLines();
|
|
fx.writeLines([...lines, record({ seq: 2, prev: JSON.parse(lines[0]).hash }, { hash: hex('9') })]);
|
|
fx.commitAll('docs(voyage): review answers (round 1)');
|
|
assertStale(fx.gate());
|
|
});
|
|
|
|
test('keep: a review record with a field beyond the record fields is stale', () => {
|
|
const fx = fixture();
|
|
const lines = fx.readLines();
|
|
const rec = JSON.parse(record({ seq: 2, prev: JSON.parse(lines[0]).hash }));
|
|
rec.note = 'merge it';
|
|
fx.writeLines([...lines, JSON.stringify(rec)]);
|
|
fx.commitAll('docs(voyage): review answers (round 1)');
|
|
assertStale(fx.gate());
|
|
});
|
|
|
|
test('keep: a review record appended beside a changed source file is stale', () => {
|
|
const fx = fixture();
|
|
fx.answer(1);
|
|
fx.put('a.mjs', 'export const a = 2;\n');
|
|
fx.commitAll('docs(voyage): review answers (round 1)');
|
|
assertStale(fx.gate());
|
|
});
|
|
|
|
test('keep: the log replaced by a symlink is stale', () => {
|
|
const fx = fixture();
|
|
rmSync(fx.logFile);
|
|
// The link points at a file tracked and unchanged since the reviewed commit,
|
|
// so the log is the only changed path and the symlink rule is what decides.
|
|
symlinkSync('../../a.mjs', fx.logFile);
|
|
assert.ok(existsSync(fx.logFile));
|
|
fx.commitAll('docs(voyage): review answers (round 1)');
|
|
// Control: the log is the only path the commit changes.
|
|
assert.deepEqual(g(fx.repo, 'diff', '--name-only', `${fx.reviewed}..HEAD`).split('\n'), [LOG_RELPATH]);
|
|
const r = fx.gate();
|
|
assertStale(r);
|
|
assert.match(r.stdout, /approvals\.jsonl is a symlink/);
|
|
});
|
|
|
|
test('keep: a valid review append to an existing log that also makes it executable is stale', () => {
|
|
const fx = fixture();
|
|
fx.answer(1);
|
|
chmodSync(fx.logFile, 0o755);
|
|
fx.commitAll('docs(voyage): review answers (round 1)');
|
|
// Control: git records the mode change.
|
|
assert.match(g(fx.repo, 'ls-tree', 'HEAD', '--', LOG_RELPATH), /^100755 /);
|
|
const r = fx.gate();
|
|
assertStale(r);
|
|
assert.match(r.stdout, NEW_REASON);
|
|
});
|
|
|
|
test('keep: a log created after the review as an executable file is stale, even with a valid review record', () => {
|
|
const fx = fixture({ seed: [] });
|
|
fx.answer(1);
|
|
chmodSync(fx.logFile, 0o755);
|
|
fx.commitAll('docs(voyage): review answers (round 1)');
|
|
// Control: the log did not exist at the reviewed commit, and is 100755 now.
|
|
assert.equal(g(fx.repo, 'ls-tree', fx.reviewed, '--', LOG_RELPATH), '');
|
|
assert.match(g(fx.repo, 'ls-tree', 'HEAD', '--', LOG_RELPATH), /^100755 /);
|
|
const r = fx.gate();
|
|
assertStale(r);
|
|
assert.match(r.stdout, NEW_REASON);
|
|
});
|
|
|
|
test('keep: a project in a subdirectory: a log under the project prefix is not the answer log', () => {
|
|
const fx = fixture({ sub: 'plugin' });
|
|
fx.put(LOG_RELPATH, `${record({ seq: 1, prev: GENESIS })}\n`);
|
|
fx.commitAll('docs(voyage): review answers (round 1)');
|
|
assertStale(fx.gate());
|
|
});
|
|
|
|
test('keep: a lessons-only commit after the review still passes', () => {
|
|
const fx = fixture();
|
|
fx.put(LESSONS, LESSONS_AFTER);
|
|
fx.put('CLAUDE.md', `# Project\n\n${POINTER}`);
|
|
fx.commitAll('docs(voyage): review lessons (round 1)');
|
|
const r = fx.gate();
|
|
assert.equal(r.status, 0, r.stdout + r.stderr);
|
|
});
|
|
|
|
// ---- [ANS] / [RLS]: fix round 4.5b2 part B, Step 4 ----
|
|
// The verdict gate and the release check keep a WARN closed until every
|
|
// counting MAJOR is answered. Every answer a tagged test needs is written by
|
|
// `review-answers.mjs record` through its CLI; the only hand-written rows are
|
|
// the ones a case is about. `keep:` cases hold at the base and stay untagged.
|
|
|
|
const ANSWERS = join(ROOT, 'lib', 'review', 'review-answers.mjs');
|
|
const RELEASE = join(ROOT, 'scripts', 'release-check.mjs');
|
|
const TRAILER = 'Co-Authored-By: Claude Test <noreply@example.invalid>';
|
|
const ANS_REL = '.claude/projects/2026-11-02-a/review.md';
|
|
const OTHER_REL = '.claude/projects/2026-11-02-b/review.md';
|
|
const ANSWERS_HEAD = '| Finding | Answer | Reason | Record | Carried from |\n|---|---|---|---|---|';
|
|
const fid = (n) => String(n).padStart(40, 'b');
|
|
const sha256 = (s) => createHash('sha256').update(s).digest('hex');
|
|
|
|
/**
|
|
* A review of `sha`. `findings` are [{severity, rule_key?, engine?}]; ids are
|
|
* fid(1), fid(2), … The trailing block's body is exactly `${json}\n`.
|
|
*/
|
|
function ansReview({ sha, verdict = 'WARN', version = '1.2', findings = [{ severity: 'MAJOR' }] }) {
|
|
const list = findings.map((f, i) => ({
|
|
id: fid(i + 1), severity: f.severity, rule_key: f.rule_key ?? 'MISSING_TEST', ...(f.engine ? { engine: f.engine } : {}),
|
|
file: 'a.mjs', line: i + 1, title: `t${i + 1}`, detail: `d${i + 1}`, recommended_action: 'Add a test.',
|
|
}));
|
|
const counts = { BLOCKER: 0, MAJOR: 0, MINOR: 0, SUGGESTION: 0 };
|
|
for (const f of list) counts[f.severity] += 1;
|
|
const json = JSON.stringify({ verdict, counts, allow_blocked_by: [], findings: list }, null, 2);
|
|
const steps = list.filter((f) => f.severity === 'BLOCKER')
|
|
.map((f, i) => `### Step R${i + 1}: fix ${f.title}\n\nSource finding: ${f.id}\n`).join('\n');
|
|
return {
|
|
json,
|
|
text: `---
|
|
type: trekreview
|
|
review_version: "${version}"
|
|
task: "t"
|
|
slug: s
|
|
project_dir: .claude/projects/2026-11-02-a/
|
|
brief_path: .claude/projects/2026-11-02-a/brief.md
|
|
scope_sha_end: ${sha}
|
|
reviewed_files_count: 1
|
|
verdict: ${verdict}
|
|
findings:
|
|
${list.map((f) => ` - ${f.id}`).join('\n')}
|
|
---
|
|
|
|
# Review: t
|
|
|
|
## Executive Summary
|
|
|
|
x
|
|
|
|
## Coverage
|
|
|
|
| File | Treatment | Reason |
|
|
|------|-----------|--------|
|
|
| a.mjs | deep-review | default |
|
|
|
|
## Remediation Summary
|
|
|
|
x
|
|
|
|
## Remediation Steps
|
|
|
|
${steps || 'None.\n'}
|
|
## Declined to Judge
|
|
|
|
None.
|
|
|
|
\`\`\`json
|
|
${json}
|
|
\`\`\`
|
|
`,
|
|
};
|
|
}
|
|
|
|
/** Put hand-written `## Answers` rows just before the trailing JSON block. */
|
|
function withRows(text, rows) {
|
|
return text.replace(/```json\n(?=[\s\S]*```\n$)/, `## Answers\n\n${ANSWERS_HEAD}\n${rows.join('\n')}\n\n\`\`\`json\n`);
|
|
}
|
|
|
|
const parse = (r) => { try { return JSON.parse(r.stdout); } catch { return {}; } };
|
|
|
|
function newRepo(prefix) {
|
|
const base = mkdtempSync(join(tmpdir(), prefix));
|
|
made.push(base);
|
|
const repo = join(base, 'repo');
|
|
mkdirSync(repo, { recursive: true });
|
|
mkdirSync(join(base, 'hooks-none'));
|
|
g(repo, 'init', '-q', '-b', 'main');
|
|
appendFileSync(join(repo, '.git', 'config'), '[user]\n\tname = Test\n\temail = test@example.invalid\n'
|
|
+ `[commit]\n\tgpgsign = false\n[core]\n\thooksPath = "${join(base, 'hooks-none')}"\n`);
|
|
// The review files are local artifacts (artifact_store: local): never tracked, never shown.
|
|
appendFileSync(join(repo, '.git', 'info', 'exclude'), '.claude/projects/\n');
|
|
return { base, repo };
|
|
}
|
|
|
|
/**
|
|
* A repo with one reviewed commit and a review of it at ANS_REL in the project
|
|
* dir (`sub` puts the project in a subdirectory). `files` adds files to the
|
|
* reviewed commit.
|
|
*/
|
|
function ansFixture({ sub = '', files = {}, ...reviewOpts } = {}) {
|
|
const { base, repo } = newRepo('review-answers-ans-');
|
|
const dir = sub ? join(repo, sub) : repo;
|
|
putIn(dir, 'a.mjs', 'export const a = 1;\n');
|
|
for (const [rel, text] of Object.entries(files)) putIn(dir, rel, text);
|
|
g(repo, 'add', '-A');
|
|
g(repo, 'commit', '-q', '-m', 'feat: the work');
|
|
const reviewed = g(repo, 'rev-parse', 'HEAD');
|
|
const reviewPath = join(dir, ...ANS_REL.split('/'));
|
|
const built = ansReview({ sha: reviewed, ...reviewOpts });
|
|
putIn(dir, ANS_REL, built.text);
|
|
const logFile = join(repo, ...LOG_RELPATH.split('/'));
|
|
const readLog = () => (existsSync(logFile) ? readFileSync(logFile, 'utf8').split('\n').filter(Boolean) : []);
|
|
return {
|
|
base, repo, dir, reviewed, reviewPath, logFile, json: built.json, readLog,
|
|
read: (p = reviewPath) => readFileSync(p, 'utf8'),
|
|
write: (text, p = reviewPath) => { mkdirSync(dirname(p), { recursive: true }); writeFileSync(p, text); },
|
|
record: ({ finding = fid(1), answer = 'accepted', reason = 'the old behaviour is kept on purpose', trailer = TRAILER, repoArg = dir, review = reviewPath } = {}) => {
|
|
const args = [ANSWERS, 'record', '--review', review, '--finding', finding, '--answer', answer,
|
|
'--repo', repoArg, '--round', '1', '--store', 'local'];
|
|
if (trailer !== null) args.push('--trailer', trailer);
|
|
return spawnSync(process.execPath, args, { input: reason, encoding: 'utf8', env: childEnv() });
|
|
},
|
|
gate: (extra = ['--repo', dir], p = reviewPath) => spawnSync(process.execPath, [GATE, '--json', p, ...extra], { encoding: 'utf8', env: childEnv() }),
|
|
/** A K2 review record for `id`, appended by the approval-log CLI and committed (hand-written rows only). */
|
|
logAnswer: (id, sha) => {
|
|
const r = spawnSync(process.execPath, [APPROVAL_LOG, 'append', '--kind', 'review', '--subject', `${ANS_REL}#${id}`,
|
|
'--sha256', sha, '--repo', repo], { cwd: dir, encoding: 'utf8', env: childEnv() });
|
|
assert.equal(r.status, 0, `approval-log append: ${r.stdout}${r.stderr}`);
|
|
g(repo, 'add', '--', LOG_RELPATH);
|
|
g(repo, 'commit', '-q', '-m', 'docs(voyage): review answers (round 1)');
|
|
return JSON.parse(r.stdout).record.hash;
|
|
},
|
|
};
|
|
}
|
|
|
|
function assertUnanswered(r, re = /^review-unanswered:/) {
|
|
assert.equal(r.status, 2, r.stdout + r.stderr);
|
|
assert.match(String(parse(r).reason), re, r.stdout);
|
|
}
|
|
|
|
test('[ANS] an unanswered MAJOR exits 2 with a reason starting review-unanswered: and the count', () => {
|
|
const fx = ansFixture({ findings: [{ severity: 'MAJOR' }, { severity: 'MINOR' }] });
|
|
const r = fx.gate();
|
|
assertUnanswered(r, /^review-unanswered: 1 MAJOR without an answer/);
|
|
assert.deepEqual(parse(r).unanswered, [fid(1)]);
|
|
assert.deepEqual(parse(r).fixed_pending, []);
|
|
});
|
|
|
|
test('[ANS] an accepted answer with a reason, bound to this review, exits 0', () => {
|
|
assert.ok(existsSync(ANSWERS), 'lib/review/review-answers.mjs must exist');
|
|
const fx = ansFixture();
|
|
const rec = fx.record({ reason: 'the old behaviour is kept on purpose' });
|
|
assert.equal(rec.status, 0, rec.stdout + rec.stderr);
|
|
const last = JSON.parse(fx.readLog().at(-1));
|
|
assert.equal(last.kind, 'review');
|
|
assert.equal(last.subject, `${ANS_REL}#${fid(1)}`);
|
|
assert.ok(fx.read().includes(`| ${fid(1)} | accepted | the old behaviour is kept on purpose | ${last.hash} | - |`), fx.read());
|
|
const r = fx.gate();
|
|
assert.equal(r.status, 0, r.stdout + r.stderr);
|
|
});
|
|
|
|
test('[ANS] a fixed answer keeps exit 2 with fixed-pending', () => {
|
|
assert.ok(existsSync(ANSWERS), 'lib/review/review-answers.mjs must exist');
|
|
const fx = ansFixture();
|
|
fx.write(fx.read().replace('## Remediation Steps\n\nNone.\n', `## Remediation Steps\n\n### Step R1: fix t1\n\nSource finding: ${fid(1)}\n`));
|
|
const rec = fx.record({ answer: 'fixed', reason: 'fixed in step R1' });
|
|
assert.equal(rec.status, 0, rec.stdout + rec.stderr);
|
|
const r = fx.gate();
|
|
assertUnanswered(r, /^review-unanswered: fixed-pending 1/);
|
|
assert.deepEqual(parse(r).fixed_pending, [fid(1)]);
|
|
});
|
|
|
|
test('[ANS] a hollow reason (sounds good) is refused by record with exit 1', () => {
|
|
assert.ok(existsSync(ANSWERS), 'lib/review/review-answers.mjs must exist');
|
|
const fx = ansFixture();
|
|
const before = fx.read();
|
|
for (const reason of ['sounds good', 'Looks good.', '', ' \n ']) {
|
|
const r = fx.record({ reason });
|
|
assert.equal(r.status, 1, `${JSON.stringify(reason)}: ${r.stdout}${r.stderr}`);
|
|
assert.equal(parse(r).recorded, false, r.stdout);
|
|
assert.deepEqual(fx.readLog(), [], `${JSON.stringify(reason)}: nothing appended`);
|
|
assert.equal(fx.read(), before, `${JSON.stringify(reason)}: review.md unchanged`);
|
|
}
|
|
});
|
|
|
|
test('[ANS] record without --trailer, or with an unfilled {model}, is refused with exit 1 and appends nothing', () => {
|
|
assert.ok(existsSync(ANSWERS), 'lib/review/review-answers.mjs must exist');
|
|
const fx = ansFixture();
|
|
const before = fx.read();
|
|
for (const trailer of [null, 'Co-Authored-By: Claude {model} <noreply@anthropic.com>']) {
|
|
const r = fx.record({ trailer });
|
|
assert.equal(r.status, 1, `${trailer}: ${r.stdout}${r.stderr}`);
|
|
assert.match(String(parse(r).reason), /--trailer/, r.stdout);
|
|
assert.deepEqual(fx.readLog(), [], `${trailer}: nothing appended`);
|
|
assert.equal(fx.read(), before, `${trailer}: review.md unchanged`);
|
|
}
|
|
});
|
|
|
|
test('[ANS] record commits the top-level log with the round message and the trailer', () => {
|
|
assert.ok(existsSync(ANSWERS), 'lib/review/review-answers.mjs must exist');
|
|
const fx = ansFixture();
|
|
const rec = fx.record();
|
|
assert.equal(rec.status, 0, rec.stdout + rec.stderr);
|
|
assert.equal(parse(rec).committed, true, rec.stdout);
|
|
const msg = g(fx.repo, 'log', '-1', '--format=%B');
|
|
assert.match(msg, /^docs\(voyage\): review answers \(round 1\)/);
|
|
assert.ok(msg.includes(TRAILER), msg);
|
|
const status = g(fx.repo, 'status', '--porcelain');
|
|
assert.ok(!status.includes('approvals.jsonl'), `the log is committed: ${status}`);
|
|
});
|
|
|
|
test('[ANS] accepted on a security MAJOR (a MAJOR-tier key with engine security-review) is refused by record', () => {
|
|
assert.ok(existsSync(ANSWERS), 'lib/review/review-answers.mjs must exist');
|
|
const fx = ansFixture({ findings: [{ severity: 'MAJOR', rule_key: 'PLACEHOLDER_IN_CODE', engine: 'security-review' }] });
|
|
const before = fx.read();
|
|
const r = fx.record({ reason: 'the risk is accepted for this release' });
|
|
assert.equal(r.status, 1, r.stdout + r.stderr);
|
|
assert.match(String(parse(r).reason), /security/, r.stdout);
|
|
assert.deepEqual(fx.readLog(), []);
|
|
assert.equal(fx.read(), before);
|
|
});
|
|
|
|
test('[ANS] a hand-written accepted row on a security MAJOR with a valid K2 record still exits 2', () => {
|
|
const fx = ansFixture({ findings: [{ severity: 'MAJOR', rule_key: 'PLACEHOLDER_IN_CODE', engine: 'security-review' }, { severity: 'MAJOR' }] });
|
|
// The binding, computed here from its definition: sha256 of the canonical JSON
|
|
// {blockSha, scopeShaEnd, findingId, answer, reason}, blockSha the sha256 of the trailing block's body.
|
|
const blockSha = sha256(`${fx.json}\n`);
|
|
const reason = 'the risk is accepted for this release';
|
|
const bind = (id) => sha256(JSON.stringify({ blockSha, scopeShaEnd: fx.reviewed, findingId: id, answer: 'accepted', reason }));
|
|
const rows = [fid(1), fid(2)].map((id) => `| ${id} | accepted | ${reason} | ${fx.logAnswer(id, bind(id))} | - |`);
|
|
fx.write(withRows(fx.read(), rows));
|
|
const r = fx.gate();
|
|
assertUnanswered(r);
|
|
// Control: the same hand-written binding answers the MAJOR that is not a security finding.
|
|
assert.deepEqual(parse(r).unanswered, [fid(1)], r.stdout);
|
|
});
|
|
|
|
test('[ANS] a reason holding |, a newline and a backtick fence is stored normalized, and the gate then exits 0', () => {
|
|
assert.ok(existsSync(ANSWERS), 'lib/review/review-answers.mjs must exist');
|
|
const fx = ansFixture();
|
|
const rec = fx.record({ reason: 'keeps the old API | callers rely on it\n```js\nfence\n```' });
|
|
assert.equal(rec.status, 0, rec.stdout + rec.stderr);
|
|
const stored = 'keeps the old API / callers rely on it js fence';
|
|
assert.equal(parse(rec).reason, stored, rec.stdout);
|
|
assert.ok(fx.read().includes(`| ${fid(1)} | accepted | ${stored} | `), fx.read());
|
|
const r = fx.gate();
|
|
assert.equal(r.status, 0, r.stdout + r.stderr);
|
|
});
|
|
|
|
test('[ANS] an answer bound to another review exits 2', () => {
|
|
assert.ok(existsSync(ANSWERS), 'lib/review/review-answers.mjs must exist');
|
|
const fx = ansFixture();
|
|
assert.equal(fx.record().status, 0);
|
|
assert.equal(fx.gate().status, 0, 'control: the answered review passes');
|
|
const other = join(fx.dir, ...OTHER_REL.split('/'));
|
|
fx.write(fx.read(), other);
|
|
assertUnanswered(fx.gate(['--repo', fx.dir], other));
|
|
});
|
|
|
|
test('[ANS] a review whose JSON block or scope_sha_end changed after the answer exits 2', () => {
|
|
assert.ok(existsSync(ANSWERS), 'lib/review/review-answers.mjs must exist');
|
|
const fx = ansFixture();
|
|
assert.equal(fx.record().status, 0);
|
|
const answered = fx.read();
|
|
assert.equal(fx.gate().status, 0, 'control: the answered review passes');
|
|
fx.write(answered.replace('"detail": "d1"', '"detail": "d1, reworded"'));
|
|
assertUnanswered(fx.gate());
|
|
// The review now names the answer-log commit (HEAD), so R6 passes and the binding decides.
|
|
const head = g(fx.repo, 'rev-parse', 'HEAD');
|
|
assert.notEqual(head, fx.reviewed);
|
|
fx.write(answered.replace(`scope_sha_end: ${fx.reviewed}`, `scope_sha_end: ${head}`));
|
|
assertUnanswered(fx.gate());
|
|
});
|
|
|
|
test('[ANS] an R-step section added above the block after the answers keeps exit 0', () => {
|
|
assert.ok(existsSync(ANSWERS), 'lib/review/review-answers.mjs must exist');
|
|
const fx = ansFixture();
|
|
assert.equal(fx.record().status, 0);
|
|
const text = fx.read();
|
|
assert.ok(text.includes('## Answers'), text);
|
|
fx.write(text.replace('## Answers', `### Step R1: fix t1\n\nSource finding: ${fid(1)}\n\n## Answers`));
|
|
const r = fx.gate();
|
|
assert.equal(r.status, 0, r.stdout + r.stderr);
|
|
});
|
|
|
|
test('[ANS] a row with no K2 record exits 2', () => {
|
|
const fx = ansFixture();
|
|
fx.write(withRows(fx.read(), [`| ${fid(1)} | accepted | the old behaviour is kept on purpose | ${hex('7')} | - |`]));
|
|
assertUnanswered(fx.gate());
|
|
});
|
|
|
|
test('[ANS] a project in a subdirectory: record --repo <subdir> writes the top-level log, and the gate exits 0 with --repo <subdir>, --repo <top level> and no --repo', () => {
|
|
assert.ok(existsSync(ANSWERS), 'lib/review/review-answers.mjs must exist');
|
|
const fx = ansFixture({ sub: 'plugin' });
|
|
const rec = fx.record({ repoArg: fx.dir });
|
|
assert.equal(rec.status, 0, rec.stdout + rec.stderr);
|
|
assert.ok(existsSync(fx.logFile), 'the top-level log is written');
|
|
assert.ok(!existsSync(join(fx.dir, ...LOG_RELPATH.split('/'))), 'no log under the project prefix');
|
|
assert.equal(JSON.parse(fx.readLog().at(-1)).subject, `plugin/${ANS_REL}#${fid(1)}`);
|
|
const status = g(fx.repo, 'status', '--porcelain');
|
|
assert.ok(!status.includes('approvals.jsonl'), `the log is committed: ${status}`);
|
|
for (const extra of [['--repo', fx.dir], ['--repo', fx.repo], []]) {
|
|
const r = fx.gate(extra);
|
|
assert.equal(r.status, 0, `${JSON.stringify(extra)}: ${r.stdout}${r.stderr}`);
|
|
}
|
|
});
|
|
|
|
test('[ANS] a 1.1 WARN review with a MAJOR exits 2', () => {
|
|
const fx = ansFixture({ version: '1.1' });
|
|
assertUnanswered(fx.gate());
|
|
});
|
|
|
|
test('[ANS] an ALLOW review over an unanswered MAJOR exits 2', () => {
|
|
const fx = ansFixture({ verdict: 'ALLOW' });
|
|
assertUnanswered(fx.gate());
|
|
});
|
|
|
|
test('[ANS] a review in a non-git directory with a MAJOR exits 2', () => {
|
|
const base = mkdtempSync(join(tmpdir(), 'review-answers-nogit-'));
|
|
made.push(base);
|
|
// Control: the directory is in no git work tree.
|
|
assert.notEqual(spawnSync('git', ['rev-parse', '--is-inside-work-tree'], { cwd: base, encoding: 'utf8', env: childEnv() }).status, 0);
|
|
const p = join(base, 'review.md');
|
|
writeFileSync(p, ansReview({ sha: 'a'.repeat(40) }).text);
|
|
const r = spawnSync(process.execPath, [GATE, '--json', p], { encoding: 'utf8', env: childEnv() });
|
|
assertUnanswered(r);
|
|
});
|
|
|
|
test('keep: a BLOCKER exits 1 whatever the answers', () => {
|
|
const fx = ansFixture({ verdict: 'BLOCK', findings: [{ severity: 'BLOCKER' }, { severity: 'MAJOR' }] });
|
|
fx.write(withRows(fx.read(), [fid(1), fid(2)].map((id) => `| ${id} | accepted | kept on purpose | ${hex('7')} | - |`)));
|
|
const r = fx.gate();
|
|
assert.equal(r.status, 1, r.stdout + r.stderr);
|
|
});
|
|
|
|
test('keep: a review of an older commit reports review-stale (not review-unanswered)', () => {
|
|
const fx = ansFixture();
|
|
putIn(fx.dir, 'a.mjs', 'export const a = 2;\n');
|
|
g(fx.repo, 'add', '-A');
|
|
g(fx.repo, 'commit', '-q', '-m', 'fix: after the review');
|
|
const r = fx.gate();
|
|
assert.equal(r.status, 2, r.stdout + r.stderr);
|
|
assert.match(r.stdout, /review-stale/);
|
|
assert.doesNotMatch(r.stdout, /review-unanswered/);
|
|
});
|
|
|
|
test('[RLS] scripts/release-check.mjs reports not ready (exit 1) for the unanswered review and ready for the answered one, from the same K2 log', () => {
|
|
assert.ok(existsSync(ANSWERS), 'lib/review/review-answers.mjs must exist');
|
|
const fx = ansFixture({
|
|
files: {
|
|
'package.json': `${JSON.stringify({ name: 'fx', version: '1.0.1', scripts: { test: 'node -e "process.exit(0)"' } }, null, 2)}\n`,
|
|
'.claude-plugin/plugin.json': `${JSON.stringify({ name: 'fx', version: '1.0.1' })}\n`,
|
|
'CHANGELOG.md': '# Changelog\n\n## v1.0.1 — 2026-11-02 — fixture\n\n### Doors\n\n- two-way: the fixture change — revert it.\n',
|
|
},
|
|
});
|
|
const other = join(fx.dir, ...OTHER_REL.split('/'));
|
|
fx.write(fx.read(), other);
|
|
const rec = fx.record();
|
|
assert.equal(rec.status, 0, rec.stdout + rec.stderr);
|
|
const check = (review) => spawnSync(process.execPath, [RELEASE, '--repo', fx.dir, '--review', review, '--json'],
|
|
{ encoding: 'utf8', env: childEnv() });
|
|
const open = check(other);
|
|
assert.equal(open.status, 1, open.stdout + open.stderr);
|
|
const rev = parse(open).checks.find((c) => c.id === 'review');
|
|
assert.equal(rev.status, 'fail', open.stdout);
|
|
assert.match(rev.detail, /review-unanswered:/);
|
|
const answered = check(fx.reviewPath);
|
|
assert.equal(answered.status, 0, answered.stdout + answered.stderr);
|
|
});
|
|
|
|
// ---- fix round 1 of Step 4 (controller findings 1 and 2) ----
|
|
// Untagged: they were red against the Step 4 build, not at the plan's base.
|
|
|
|
test('a review with no readable trailing JSON block is not measured: decideAnswers is not ok and decideOpenFindings exits 2', async () => {
|
|
assert.ok(existsSync(ANSWERS), 'lib/review/review-answers.mjs must exist');
|
|
const { decideAnswers } = await import(ANSWERS);
|
|
const { decideOpenFindings } = await import(GATE);
|
|
const { text, json } = ansReview({ sha: 'a'.repeat(40) });
|
|
const block = `\`\`\`json\n${json}\n\`\`\`\n`;
|
|
assert.ok(text.endsWith(block), 'control: the fixture ends with its JSON block');
|
|
const head = text.slice(0, -block.length);
|
|
const cases = {
|
|
'no block': head,
|
|
'an unclosed block': `${head}\`\`\`json\n${json}\n`,
|
|
'a non-json last block': `${text}\n\`\`\`yaml\nverdict: WARN\n\`\`\`\n`,
|
|
'an unparseable block': `${head}\`\`\`json\n{ "verdict": "WARN", \n\`\`\`\n`,
|
|
};
|
|
const base = mkdtempSync(join(tmpdir(), 'review-answers-unread-'));
|
|
made.push(base);
|
|
const reviewPath = join(base, 'review.md');
|
|
for (const [name, t] of Object.entries(cases)) {
|
|
const d = decideAnswers(t, { records: [], reviewSubject: 'review.md' });
|
|
assert.equal(d.ok, false, `${name}: decideAnswers must not be ok`);
|
|
assert.match(String(d.reason), /^review-not-measured:/, `${name}: ${d.reason}`);
|
|
assert.deepEqual(d.unanswered, [], name);
|
|
assert.deepEqual(d.fixedPending, [], name);
|
|
const o = decideOpenFindings(t, { reviewPath });
|
|
assert.equal(o.exit, 2, `${name}: decideOpenFindings must exit 2, got ${o.exit} (${o.reason})`);
|
|
assert.match(String(o.reason), /^review-not-measured:/, `${name}: ${o.reason}`);
|
|
}
|
|
});
|
|
|
|
test('record on a review with a malformed ## Answers table exits 2 and appends nothing to the approval log', () => {
|
|
assert.ok(existsSync(ANSWERS), 'lib/review/review-answers.mjs must exist');
|
|
const fx = ansFixture();
|
|
const malformed = fx.read().replace(/```json\n(?=[\s\S]*```\n$)/,
|
|
'## Answers\n\n| Finding | Answer |\n|---|---|\n\n```json\n');
|
|
assert.notEqual(malformed, fx.read(), 'control: the malformed section is in the review');
|
|
fx.write(malformed);
|
|
const r = fx.record();
|
|
assert.equal(r.status, 2, r.stdout + r.stderr);
|
|
assert.equal(parse(r).recorded, false, r.stdout);
|
|
assert.deepEqual(fx.readLog(), [], 'the approval log gained no line');
|
|
assert.equal(fx.read(), malformed, 'review.md unchanged');
|
|
});
|