open/voyage
1
0
Fork 0
Code Issues Releases 13 Activity

test(red-proof): the tests the 4.5a discovered-error fix added carry their id tags

Browse source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-10-26 17:45:46 +01:00
commit b9d173a7fd
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
3 changed files with 128 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

84
tests/fix-round-4-5b-mutations.mjs
View file

@ -150,4 +150,88 @@ run_tagged "$TMP/base.jsonl" --test-timeout=300000 --test-force-exit $PATS`,
},
],
},
{
key: 'M1',
id: 'M1',
what: 'the 13 name sites of the tests f6bdc2d added are put back to their untagged found in execution: names',
spans: [
{
file: 'tests/lib/fix-round-4-5a-prose.test.mjs',
find: '[WF] sibling: ${rel} no longer puts back what the coordinator did not return',
replace: 'found in execution: ${rel} no longer puts back what the coordinator did not return',
},
{
file: 'tests/lib/fix-round-4-5a-prose.test.mjs',
find: '[WF] sibling: ${rel} says a finding is matched to its row',
replace: 'found in execution: ${rel} says a finding is matched to its row',
},
{
file: 'tests/lib/fix-round-4-5a-prose.test.mjs',
find: '[WF] sibling: ${rel} puts back one copy per row',
replace: 'found in execution: ${rel} puts back one copy per row',
},
{
file: 'tests/lib/fix-round-4-5a-prose.test.mjs',
find: "[WF] sibling: ${rel} says floor (b)'s counterpart",
replace: "found in execution: ${rel} says floor (b)'s counterpart",
},
{
file: 'tests/lib/fix-round-4-5a-prose.test.mjs',
find: '[M1r] README.md names no "leading recommendation" trigger',
replace: 'found in execution: README.md names no "leading recommendation" trigger',
},
{
file: 'tests/lib/fix-round-4-5a-prose.test.mjs',
find: '[#113] sibling: README.md stars security-researcher nowhere',
replace: 'found in execution: README.md stars security-researcher nowhere',
},
{
file: 'tests/lib/fix-round-4-5a-prose.test.mjs',
find: '[M1r] README.md lists security-researcher as no conditional agent',
replace: 'found in execution: README.md lists security-researcher as no conditional agent',
},
{
file: 'tests/lib/fix-round-4-5a-prose.test.mjs',
find: '[#113] sibling: README.md names the deep-research case of wave 2',
replace: 'found in execution: README.md names the deep-research case of wave 2',
},
{
file: 'tests/lib/fix-round-4-5a-prose.test.mjs',
find: '[#113] sibling: README.md runs contrarian-researcher after wave 1 returns',
replace: 'found in execution: README.md runs contrarian-researcher after wave 1 returns',
},
{
file: 'tests/lib/fix-round-4-5a-prose.test.mjs',
find: "[#111] sibling: ${rel} item 3 names Pass 3's high-effort skip",
replace: "found in execution: ${rel} item 3 names Pass 3's high-effort skip",
},
{
file: 'tests/lib/fix-round-4-5a-prose.test.mjs',
find: '[#111] sibling: CHANGELOG ### Fixed has the **Across the round** label',
replace: 'found in execution: CHANGELOG ### Fixed has the **Across the round** label',
},
{
file: 'tests/lib/fix-round-4-5a-prose.test.mjs',
find: '[#111] sibling: commands/trekreview.md high-effort section skips Pass 3',
replace: 'found in execution: commands/trekreview.md high-effort section skips Pass 3',
},
{
file: 'tests/lib/fix-round-4-5a-prose.test.mjs',
find: '[#111] sibling: agents/review-orchestrator.md quick mode skips pass 3',
replace: 'found in execution: agents/review-orchestrator.md quick mode skips pass 3',
},
],
},
{
key: 'M1r',
id: 'M1r',
what: 'the README * = conditional line is put back to its 8a2a989 text, which names security-researcher when a leading recommendation forms',
spans: [
{
file: 'README.md',
find: "`* = conditional`: convention-scanner / test-strategist on medium+ codebases (50+ files); research-scout for unknown external tech; contrarian-researcher, when selected, runs in wave 2 after wave 1 returns, with the leading answer of wave 1's external findings to attack (with `--engine deep-research`, after the engine returns, from the leading answer of its report), always at `effort=high`; brief-conformance-reviewer skipped under `--quick`.",
replace: '`* = conditional`: convention-scanner / test-strategist on medium+ codebases (50+ files); research-scout for unknown external tech; security-researcher when a leading recommendation forms (or always at `effort=high`); contrarian-researcher, when selected, runs in wave 2 after the other external agents return, with their leading answer to attack (always at `effort=high`); brief-conformance-reviewer skipped under `--quick`.',
},
],
},
];

44
tests/lib/fix-round-4-5a-prose.test.mjs
View file

@ -7,8 +7,9 @@
// assertion at ab6b32c (the round's start point), except DJ's pins: they pass
// there by design, and tests/fix-round-4-5a-red.sh proves them red by mutation.
// Untagged tests (prefixes `helpers:`, `step 25:`, `keep:`, `control:`,
// `arch:`, `e2e script:`, `red script:`, `CHANGELOG:`, `found in execution:`)
// are outside the round's id list.
// `arch:`, `e2e script:`, `red script:`, `CHANGELOG:`) are outside the
// round's id list; a `[M1r]` test carries a 4.5b id and is that round's to
// judge.
// #23 code-correctness-reviewer cites a "MANDATORY OVERRIDE rule above"
// that is no longer there; the sentence says the rule itself.
// #24 research-scout says "Try Tavily first" but its tools held no Tavily
@ -1758,10 +1759,13 @@ test('CHANGELOG: the released v6.5.0 section is byte-equal to its text at ab6b32
'the released v6.5.0 section changed; edit only the round\'s section');
});
// Found in execution (untagged, outside the round's id list): texts next to
// the round's fixes that still said what the code no longer does. Each test
// fails on an assertion at 8a2a989, before the text changed; its presence
// assertion holds there.
// Found in execution (f6bdc2d): texts next to the round's fixes that still
// said what the code no longer does. Each test fails on an assertion at
// 8a2a989, before the text changed; its presence assertion holds there. Each
// carries its 4.5a parent tag (`[WF] sibling:`, `[#113] sibling:`,
// `[#111] sibling:`) and fails on an assertion at ab6b32c, except the two
// `[M1r]` tests: they pass at ab6b32c, so they carry a 4.5b id and
// tests/fix-round-4-5b-red.sh proves them red under the M1r mutation.
// D1: the workflow matches each finding to its own row (`rowOf` in
// scripts/trekreview-armB.workflow.mjs), not to its exact triplet.
@ -1774,25 +1778,25 @@ const COUNTERPART_IS = 'its counterpart is its own row, else the strictest row o
const PUT_BACK_IS = 'of the engine findings in a row no returned finding is on, the one with the highest severity (on a tie, the first seen) is put back at its own severity (operator decision 23)';
for (const [rel, part] of PUT_BACK_TEXTS) {
test(`found in execution: ${rel} no longer puts back what the coordinator did not return "at its exact (file, line, rule_key)"`, () => {
test(`[WF] sibling: ${rel} no longer puts back what the coordinator did not return "at its exact (file, line, rule_key)"`, () => {
const text = part();
assert.ok(text.includes('put back at its own severity'), `${rel}: the put-back sentence is missing: ${text}`);
assert.doesNotMatch(text, /at its exact `\(file, line, rule_key\)`/, `${rel}: the put-back still matches on the exact triplet`);
});
test(`found in execution: ${rel} says a finding is matched to its row, the triplet plus the normalized title`, () => {
test(`[WF] sibling: ${rel} says a finding is matched to its row, the triplet plus the normalized title`, () => {
const text = part();
assert.ok(text.includes('put back at its own severity'), `${rel}: the put-back sentence is missing: ${text}`);
assert.ok(text.includes(`each finding is matched to its own row, ${ROW_IS}`), `${rel}: the put-back does not say the row: ${text}`);
});
test(`found in execution: ${rel} puts back one copy per row, the row's engine finding with the highest severity`, () => {
test(`[WF] sibling: ${rel} puts back one copy per row, the row's engine finding with the highest severity`, () => {
const text = part();
assert.ok(text.includes('put back at its own severity'), `${rel}: the put-back sentence is missing: ${text}`);
assert.ok(text.includes(PUT_BACK_IS), `${rel}: the put-back does not say which one copy of a row is put back: ${text}`);
});
test(`found in execution: ${rel} says floor (b)'s counterpart is the finding's own row, else the strictest row on its triplet`, () => {
test(`[WF] sibling: ${rel} says floor (b)'s counterpart is the finding's own row, else the strictest row on its triplet`, () => {
const text = part();
assert.ok(text.includes('(floor (b))'), `${rel}: floor (b) is missing: ${text}`);
assert.ok(text.includes(COUNTERPART_IS), `${rel}: floor (b) does not say which row is the counterpart: ${text}`);
@ -1802,31 +1806,31 @@ for (const [rel, part] of PUT_BACK_TEXTS) {
// D2: the README's conditional line says what /trekresearch says.
const conditionalLine = () => partAt('README.md', '`* = conditional`:', () => true);
test('found in execution: README.md names no "leading recommendation" trigger for security-researcher', () => {
test('[M1r] README.md names no "leading recommendation" trigger for security-researcher', () => {
const text = conditionalLine();
assert.ok(text.includes('contrarian-researcher'), `the conditional line does not name contrarian-researcher: ${text}`);
assert.doesNotMatch(text, /security-researcher when a leading recommendation forms/, 'README.md still names a trigger /trekresearch does not define');
});
test('found in execution: README.md stars security-researcher nowhere, as docs-researcher and community-researcher (selected by scope alike)', () => {
test('[#113] sibling: README.md stars security-researcher nowhere, as docs-researcher and community-researcher (selected by scope alike)', () => {
const diagram = partAt('README.md', 'RE["EXTERNAL: docs-researcher', () => true);
assert.ok(diagram.includes('security-researcher'), `the diagram's external line does not name security-researcher: ${diagram}`);
assert.ok(diagram.includes('security-researcher · contrarian-researcher*'), `the diagram still stars security-researcher: ${diagram}`);
});
test('found in execution: README.md lists security-researcher as no conditional agent', () => {
test('[M1r] README.md lists security-researcher as no conditional agent', () => {
const text = conditionalLine();
assert.ok(text.includes('contrarian-researcher'), `the conditional line does not name contrarian-researcher: ${text}`);
assert.ok(!text.includes('security-researcher'), `the conditional line still lists security-researcher: ${text}`);
});
test('found in execution: README.md names the deep-research case of wave 2', () => {
test('[#113] sibling: README.md names the deep-research case of wave 2', () => {
const text = conditionalLine();
assert.ok(text.includes('contrarian-researcher, when selected, runs in wave 2'), `the conditional line does not name wave 2: ${text}`);
assert.ok(text.includes('contrarian-researcher, when selected, runs in wave 2 after wave 1 returns, with the leading answer of wave 1\'s external findings to attack (with `--engine deep-research`, when the engine is active, after the engine returns, from the leading answer of its report), always at `effort=high`;'), `README.md does not name the deep-research case of wave 2: ${text}`);
});
test('found in execution: README.md runs contrarian-researcher after wave 1 returns', () => {
test('[#113] sibling: README.md runs contrarian-researcher after wave 1 returns', () => {
const text = conditionalLine();
assert.ok(text.includes('contrarian-researcher, when selected, runs in wave 2'), `the conditional line does not name wave 2: ${text}`);
assert.ok(text.includes('runs in wave 2 after wave 1 returns'), `README.md does not say "after wave 1 returns": ${text}`);
@ -1840,15 +1844,15 @@ const HIGH_EFFORT_SKIP = {
};
for (const rel of Object.keys(HIGH_EFFORT_SKIP)) {
test(`found in execution: ${rel} item 3 names Pass 3's high-effort skip and the PLAN_EXECUTE_DRIFT normalization`, () => {
test(`[#111] sibling: ${rel} item 3 names Pass 3's high-effort skip and the PLAN_EXECUTE_DRIFT normalization`, () => {
const item = pass3Item(rel);
assert.ok(item.includes(PASS3_ITEM.slice(PASS3_ITEM.indexOf(' — ') + 3)), `${rel}: item 3 is not the Pass 3 sentence: ${item}`);
assert.match(item, HIGH_EFFORT_SKIP[rel], `${rel}: item 3 does not name the high-effort skip: ${item}`);
assert.ok(item.includes(PASS3_ITEM.slice(PASS3_ITEM.indexOf(' — ') + 3)), `${rel}: item 3 is not the Pass 3 sentence: ${item}`);
});
}
// D4: the siblings entry has its own group label.
test('found in execution: CHANGELOG ### Fixed has the **Across the round** label directly above the siblings entry', () => {
test('[#111] sibling: CHANGELOG ### Fixed has the **Across the round** label directly above the siblings entry', () => {
const lines = section(roundPart(), '### Fixed').split('\n');
const at = lines.map((l, i) => (l.startsWith('- **The texts next to the fixes say the same**') ? i : -1)).filter((i) => i !== -1);
assert.equal(at.length, 1, `### Fixed must hold the siblings entry once; found ${at.length}`);
@ -1861,14 +1865,14 @@ test('found in execution: CHANGELOG ### Fixed has the **Across the round** label
// "still applies in quick mode").
const BRIEF_CONFLICT_KEPT = ', except the brief-conflict rule, which applies in every mode.';
test('found in execution: commands/trekreview.md high-effort section skips Pass 3 except the brief-conflict rule', () => {
test('[#111] sibling: commands/trekreview.md high-effort section skips Pass 3 except the brief-conflict rule', () => {
const text = flat(section(read('commands/trekreview.md'), '### High-effort behavior (v5.1.1)'));
assert.ok(text.includes('skip Pass 3 (Cloudflare reasonableness filter)'), `the high-effort section does not skip Pass 3: ${text}`);
assert.ok(text.includes(`skip Pass 3 (Cloudflare reasonableness filter) in \`agents/review-coordinator.md\`${BRIEF_CONFLICT_KEPT}`),
`the high-effort skip keeps no brief-conflict rule: ${text}`);
});
test('found in execution: agents/review-orchestrator.md quick mode skips pass 3 except the brief-conflict rule', () => {
test('[#111] sibling: agents/review-orchestrator.md quick mode skips pass 3 except the brief-conflict rule', () => {
const text = partAt('agents/review-orchestrator.md', 'In `quick` mode, skip pass 3', toBlank);
assert.ok(text.includes('skip pass 3 (reasonableness filter)'), `the quick-mode line does not skip pass 3: ${text}`);
assert.ok(text.includes(`In \`quick\` mode, skip pass 3 (reasonableness filter)${BRIEF_CONFLICT_KEPT}`),

20
tests/lib/fix-round-4-5b-prose.test.mjs
View file

@ -45,6 +45,9 @@
// E18a The F9 deep-review and summary-only rows are pinned exactly (red under
// the E18a mutation).
// Db No README line stars security-researcher (red under the Db mutation).
// M1 No test name in the 4.5a prose file starts with `found in execution:`;
// the tests f6bdc2d added carry their 4.5a parent tag (red under the
// M1 mutation, which puts the names back).
// No static import of a new export: the file must load at be0b631.
// Fix the SOURCE a pin guards, not the pin.
@ -483,3 +486,20 @@ test('[Db] the README stars security-researcher nowhere', () => {
assert.match(readme, /security-researcher/, 'README.md names security-researcher nowhere');
assert.doesNotMatch(readme, /security-researcher\*/, 'a README line stars security-researcher as conditional');
});
// M1: the tests the 4.5a discovered-error fix (f6bdc2d) added carry their id
// tags, so a red script judges them; none is left under the untagged
// `found in execution:` prefix.
const PROSE_4_5A = 'tests/lib/fix-round-4-5a-prose.test.mjs';
/** The name text after each `test(` call opener in `rel`: the start of every test name, templates included. */
function testNameStarts(rel) {
return [...read(rel).matchAll(/\btest\(\s*(['"`])([^\n]*)/g)].map((m) => m[2]);
}
test('[M1] no test name in the 4.5a prose file starts with found in execution:', () => {
const names = testNameStarts(PROSE_4_5A);
assert.ok(names.length > 0, `${PROSE_4_5A} holds no test name`);
const untagged = names.filter((n) => n.startsWith('found in execution:'));
assert.deepEqual(untagged, [], `${PROSE_4_5A} still names ${untagged.length} test(s) "found in execution:"`);
});